@livestore/sqlite-wasm 0.0.0-snapshot-1d99fea7d2ce2c7a5d9ed0a3752f8a7bda6bc3db

package/src/make-sqlite-db.ts

@@ -0,0 +1,220 @@
+import type {
+  PersistenceInfo,
+  PreparedBindValues,
+  PreparedStatement,
+  SqliteDb,
+  SqliteDbChangeset,
+} from '@livestore/common'
+import { SqliteError } from '@livestore/common'
+import * as SqliteConstants from '@livestore/wa-sqlite/src/sqlite-constants.js'
+
+import { makeInMemoryDb } from './in-memory-vfs.js'
+
+export const makeSqliteDb = <
+  TMetadata extends {
+    dbPointer: number
+    persistenceInfo: PersistenceInfo
+    deleteDb: () => void
+    configureDb: (db: SqliteDb<TMetadata>) => void
+  },
+>({
+  sqlite3,
+  metadata,
+}: {
+  sqlite3: SQLiteAPI
+  metadata: TMetadata
+}): SqliteDb<TMetadata> => {
+  const preparedStmts: PreparedStatement[] = []
+  const { dbPointer } = metadata
+
+  let isClosed = false
+
+  const sqliteDb: SqliteDb<TMetadata> = {
+    _tag: 'SqliteDb',
+    metadata,
+    prepare: (queryStr) => {
+      try {
+        const stmts = sqlite3.statements(dbPointer, queryStr.trim(), { unscoped: true })
+
+        let isFinalized = false
+
+        const preparedStmt = {
+          execute: (bindValues, options) => {
+            for (const stmt of stmts) {
+              if (bindValues !== undefined && Object.keys(bindValues).length > 0) {
+                sqlite3.bind_collection(stmt, bindValues as any)
+              }
+
+              try {
+                sqlite3.step(stmt)
+              } finally {
+                if (options?.onRowsChanged) {
+                  options.onRowsChanged(sqlite3.changes(dbPointer))
+                }
+
+                sqlite3.reset(stmt) // Reset is needed for next execution
+              }
+            }
+          },
+          select: <T>(bindValues: PreparedBindValues) => {
+            if (stmts.length !== 1) {
+              throw new SqliteError({
+                query: { bindValues, sql: queryStr },
+                code: -1,
+                cause: 'Expected only one statement when using `select`',
+              })
+            }
+
+            const stmt = stmts[0]!
+
+            if (bindValues !== undefined && Object.keys(bindValues).length > 0) {
+              sqlite3.bind_collection(stmt, bindValues as any)
+            }
+
+            const results: T[] = []
+
+            try {
+              // NOTE `column_names` only works for `SELECT` statements, ignoring other statements for now
+              let columns = undefined
+              try {
+                columns = sqlite3.column_names(stmt)
+                // eslint-disable-next-line @typescript-eslint/no-unused-vars
+              } catch (_e) {}
+
+              while (sqlite3.step(stmt) === SqliteConstants.SQLITE_ROW) {
+                if (columns !== undefined) {
+                  const obj: { [key: string]: any } = {}
+                  for (let i = 0; i < columns.length; i++) {
+                    obj[columns[i]!] = sqlite3.column(stmt, i)
+                  }
+                  results.push(obj as unknown as T)
+                }
+              }
+            } catch (e) {
+              throw new SqliteError({
+                query: { bindValues, sql: queryStr },
+                code: (e as any).code,
+                cause: e,
+              })
+            } finally {
+              // reset the cached statement so we can use it again in the future
+              sqlite3.reset(stmt)
+            }
+
+            return results
+          },
+          finalize: () => {
+            // Avoid double finalization which leads to a crash
+            if (isFinalized) {
+              return
+            }
+
+            isFinalized = true
+
+            for (const stmt of stmts) {
+              sqlite3.finalize(stmt)
+            }
+          },
+          sql: queryStr,
+        } satisfies PreparedStatement
+
+        preparedStmts.push(preparedStmt)
+
+        return preparedStmt
+      } catch (e) {
+        throw new SqliteError({
+          query: { sql: queryStr, bindValues: {} },
+          code: (e as any).code,
+          cause: e,
+        })
+      }
+    },
+    export: () => sqlite3.serialize(dbPointer, 'main'),
+    execute: (queryStr, bindValues, options) => {
+      const stmt = sqliteDb.prepare(queryStr)
+      stmt.execute(bindValues, options)
+      stmt.finalize()
+    },
+    select: (queryStr, bindValues) => {
+      const stmt = sqliteDb.prepare(queryStr)
+      const results = stmt.select(bindValues)
+      stmt.finalize()
+      return results as ReadonlyArray<any>
+    },
+    destroy: () => {
+      sqliteDb.close()
+
+      metadata.deleteDb()
+      // if (metadata._tag === 'opfs') {
+      //   metadata.vfs.resetAccessHandle(metadata.fileName)
+      // }
+    },
+    close: () => {
+      if (isClosed) {
+        return
+      }
+
+      for (const stmt of preparedStmts) {
+        stmt.finalize()
+      }
+      sqlite3.close(dbPointer)
+      isClosed = true
+    },
+    import: (source) => {
+      // https://www.sqlite.org/c3ref/c_deserialize_freeonclose.html
+      // #define SQLITE_DESERIALIZE_FREEONCLOSE 1 /* Call sqlite3_free() on close */
+      // #define SQLITE_DESERIALIZE_RESIZEABLE  2 /* Resize using sqlite3_realloc64() */
+      // #define SQLITE_DESERIALIZE_READONLY    4 /* Database is read-only */
+      const FREE_ON_CLOSE = 1
+      const RESIZEABLE = 2
+
+      // NOTE in case we'll have a future use-case where we need a read-only database, we can reuse this code below
+      // if (readOnly === true) {
+      //   sqlite3.deserialize(db, 'main', bytes, bytes.length, bytes.length, FREE_ON_CLOSE | RESIZEABLE)
+      // } else {
+      if (source instanceof Uint8Array) {
+        const tmpDb = makeInMemoryDb(sqlite3)
+        // TODO find a way to do this more efficiently with sqlite to avoid either of the deserialize + backup call
+        // Maybe this can be done via the VFS API
+        sqlite3.deserialize(tmpDb.dbPointer, 'main', source, source.length, source.length, FREE_ON_CLOSE | RESIZEABLE)
+        sqlite3.backup(dbPointer, 'main', tmpDb.dbPointer, 'main')
+        sqlite3.close(tmpDb.dbPointer)
+      } else {
+        sqlite3.backup(dbPointer, 'main', source.metadata.dbPointer, 'main')
+      }
+
+      metadata.configureDb(sqliteDb)
+    },
+    session: () => {
+      const sessionPointer = sqlite3.session_create(dbPointer, 'main')
+      sqlite3.session_attach(sessionPointer, null)
+
+      return {
+        changeset: () => {
+          const res = sqlite3.session_changeset(sessionPointer)
+          return res.changeset ?? undefined
+        },
+        finish: () => {
+          sqlite3.session_delete(sessionPointer)
+        },
+      }
+    },
+    makeChangeset: (data) => {
+      const changeset = {
+        invert: () => {
+          const inverted = sqlite3.changeset_invert(data)
+          return sqliteDb.makeChangeset(inverted)
+        },
+        apply: () => {
+          sqlite3.changeset_apply(dbPointer, data)
+        },
+      } satisfies SqliteDbChangeset
+
+      return changeset
+    },
+  } satisfies SqliteDb<TMetadata>
+
+  metadata.configureDb(sqliteDb)
+
+  return sqliteDb
+}

package/src/node/NodeFS.ts

@@ -0,0 +1,190 @@
+/// <reference types="node" />
+
+/* eslint-disable prefer-arrow/prefer-arrow-functions */
+import * as fs from 'node:fs'
+import path from 'node:path'
+
+import type * as WaSqlite from '@livestore/wa-sqlite'
+import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
+
+import { FacadeVFS } from '../FacadeVFS.js'
+
+interface NodeFsFile {
+  pathname: string
+  flags: number
+  fileHandle: number | null
+}
+
+export class NodeFS extends FacadeVFS {
+  private mapIdToFile = new Map<number, NodeFsFile>()
+  private lastError: Error | null = null
+  private readonly directory: string
+  constructor(name: string, sqlite3: WaSqlite.SQLiteAPI, directory: string) {
+    super(name, sqlite3)
+
+    this.directory = directory
+  }
+
+  getFilename(fileId: number): string {
+    const pathname = this.mapIdToFile.get(fileId)?.pathname
+    return `NodeFS:${pathname}`
+  }
+
+  jOpen(zName: string | null, fileId: number, flags: number, pOutFlags: DataView): number {
+    try {
+      const pathname = zName ? path.resolve(this.directory, zName) : Math.random().toString(36).slice(2)
+      const file: NodeFsFile = { pathname, flags, fileHandle: null }
+      this.mapIdToFile.set(fileId, file)
+
+      const create = !!(flags & VFS.SQLITE_OPEN_CREATE)
+      const readwrite = !!(flags & VFS.SQLITE_OPEN_READWRITE)
+
+      // Convert SQLite flags to Node.js flags
+      let fsFlags = 'r'
+      if (create && readwrite) {
+        // Check if file exists first
+        const exists = fs.existsSync(pathname)
+        fsFlags = exists ? 'r+' : 'w+' // Use r+ for existing files, w+ only for new files
+      } else if (readwrite) {
+        fsFlags = 'r+' // Open file for reading and writing
+      }
+
+      try {
+        file.fileHandle = fs.openSync(pathname, fsFlags)
+        pOutFlags.setInt32(0, flags, true)
+        return VFS.SQLITE_OK
+      } catch (err: any) {
+        if (err.code === 'ENOENT' && !create) {
+          return VFS.SQLITE_CANTOPEN
+        }
+        throw err
+      }
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_CANTOPEN
+    }
+  }
+
+  jRead(fileId: number, pData: Uint8Array, iOffset: number): number {
+    try {
+      const file = this.mapIdToFile.get(fileId)
+      if (!file?.fileHandle) return VFS.SQLITE_IOERR_READ
+
+      // const view = new DataView(pData.buffer, pData.byteOffset, pData.length)
+      // const bytesRead = fs.readSync(file.fileHandle, view, 0, pData.length, iOffset)
+      const bytesRead = fs.readSync(file.fileHandle, pData.subarray(), { position: iOffset })
+
+      if (bytesRead < pData.length) {
+        pData.fill(0, bytesRead)
+        return VFS.SQLITE_IOERR_SHORT_READ
+      }
+      return VFS.SQLITE_OK
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_IOERR_READ
+    }
+  }
+
+  jWrite(fileId: number, pData: Uint8Array, iOffset: number): number {
+    try {
+      const file = this.mapIdToFile.get(fileId)
+      if (!file?.fileHandle) return VFS.SQLITE_IOERR_WRITE
+
+      // const view = new DataView(pData.buffer, pData.byteOffset, pData.length)
+      // fs.writeSync(file.fileHandle, view, 0, pData.length, iOffset)
+      fs.writeSync(file.fileHandle, Buffer.from(pData.subarray()), 0, pData.length, iOffset)
+      return VFS.SQLITE_OK
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_IOERR_WRITE
+    }
+  }
+
+  jClose(fileId: number): number {
+    try {
+      const file = this.mapIdToFile.get(fileId)
+      if (!file) return VFS.SQLITE_OK
+
+      this.mapIdToFile.delete(fileId)
+      if (file.fileHandle !== null) {
+        fs.closeSync(file.fileHandle)
+      }
+
+      if (file.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
+        fs.unlinkSync(file.pathname)
+      }
+      return VFS.SQLITE_OK
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_IOERR_CLOSE
+    }
+  }
+
+  jFileSize(fileId: number, pSize64: DataView): number {
+    try {
+      const file = this.mapIdToFile.get(fileId)
+      if (!file?.fileHandle) return VFS.SQLITE_IOERR_FSTAT
+
+      const stats = fs.fstatSync(file.fileHandle)
+      pSize64.setBigInt64(0, BigInt(stats.size), true)
+      return VFS.SQLITE_OK
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_IOERR_FSTAT
+    }
+  }
+
+  jTruncate(fileId: number, iSize: number): number {
+    try {
+      const file = this.mapIdToFile.get(fileId)
+      if (!file?.fileHandle) return VFS.SQLITE_IOERR_TRUNCATE
+
+      fs.ftruncateSync(file.fileHandle, iSize)
+      return VFS.SQLITE_OK
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_IOERR_TRUNCATE
+    }
+  }
+
+  jSync(fileId: number, _flags: number): number {
+    try {
+      const file = this.mapIdToFile.get(fileId)
+      if (!file?.fileHandle) return VFS.SQLITE_OK
+
+      // TODO do this out of band (for now we disable it to speed up the node vfs)
+      // fs.fsyncSync(file.fileHandle)
+      return VFS.SQLITE_OK
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_IOERR_FSYNC
+    }
+  }
+
+  jDelete(zName: string, _syncDir: number): number {
+    try {
+      const pathname = path.resolve(this.directory, zName)
+      fs.unlinkSync(pathname)
+      return VFS.SQLITE_OK
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_IOERR_DELETE
+    }
+  }
+
+  jAccess(zName: string, _flags: number, pResOut: DataView): number {
+    try {
+      const pathname = path.resolve(this.directory, zName)
+      const exists = fs.existsSync(pathname)
+      pResOut.setInt32(0, exists ? 1 : 0, true)
+      return VFS.SQLITE_OK
+    } catch (e: any) {
+      this.lastError = e
+      return VFS.SQLITE_IOERR_ACCESS
+    }
+  }
+
+  deleteDb(fileName: string) {
+    fs.unlinkSync(path.join(this.directory, fileName))
+  }
+}

package/src/node/mod.ts

@@ -0,0 +1,132 @@
+import path from 'node:path'
+
+import { type MakeSqliteDb, type PersistenceInfo, type SqliteDb, UnexpectedError } from '@livestore/common'
+import { Effect, FileSystem } from '@livestore/utils/effect'
+import type * as WaSqlite from '@livestore/wa-sqlite'
+import type { MemoryVFS } from '@livestore/wa-sqlite/src/examples/MemoryVFS.js'
+
+import { makeInMemoryDb } from '../in-memory-vfs.js'
+import { makeSqliteDb } from '../make-sqlite-db.js'
+import { NodeFS } from './NodeFS.js'
+
+export type NodeDatabaseMetadataInMemory = {
+  _tag: 'in-memory'
+  vfs: MemoryVFS
+  dbPointer: number
+  persistenceInfo: PersistenceInfo
+  deleteDb: () => void
+  configureDb: (db: SqliteDb) => void
+}
+
+export type NodeDatabaseMetadataFs = {
+  _tag: 'fs'
+  vfs: NodeFS
+  dbPointer: number
+  persistenceInfo: PersistenceInfo<{ directory: string }>
+  deleteDb: () => void
+  configureDb: (db: SqliteDb) => void
+}
+
+export type NodeDatabaseMetadata = NodeDatabaseMetadataInMemory | NodeDatabaseMetadataFs
+
+export type NodeDatabaseInputInMemory = {
+  _tag: 'in-memory'
+  configureDb?: (db: SqliteDb) => void
+}
+
+export type NodeDatabaseInputFs = {
+  _tag: 'fs'
+  directory: string
+  fileName: string
+  configureDb?: (db: SqliteDb) => void
+}
+
+export type NodeDatabaseInput = NodeDatabaseInputInMemory | NodeDatabaseInputFs
+
+export const sqliteDbFactory = ({
+  sqlite3,
+}: {
+  sqlite3: SQLiteAPI
+}): Effect.Effect<
+  MakeSqliteDb<{ dbPointer: number; persistenceInfo: PersistenceInfo }, NodeDatabaseInput, NodeDatabaseMetadata>,
+  never,
+  FileSystem.FileSystem
+> =>
+  Effect.andThen(
+    FileSystem.FileSystem,
+    (fs) => (input) =>
+      Effect.gen(function* () {
+        if (input._tag === 'in-memory') {
+          const { dbPointer, vfs } = makeInMemoryDb(sqlite3)
+          return makeSqliteDb<NodeDatabaseMetadataInMemory>({
+            sqlite3,
+            metadata: {
+              _tag: 'in-memory',
+              vfs,
+              dbPointer,
+              persistenceInfo: { fileName: ':memory:' },
+              deleteDb: () => {},
+              configureDb: input.configureDb ?? (() => {}),
+            },
+          }) as any
+        }
+
+        const { dbPointer, vfs } = yield* makeNodeFsDb({
+          sqlite3,
+          fileName: input.fileName,
+          directory: input.directory,
+          fs,
+        })
+
+        const filePath = path.join(input.directory, input.fileName)
+
+        return makeSqliteDb<NodeDatabaseMetadataFs>({
+          sqlite3,
+          metadata: {
+            _tag: 'fs',
+            vfs,
+            dbPointer,
+            persistenceInfo: { fileName: input.fileName, directory: input.directory },
+            deleteDb: () => vfs.deleteDb(filePath),
+            configureDb: input.configureDb ?? (() => {}),
+          },
+        })
+      }),
+  )
+
+let nodeFsVfs: NodeFS | undefined
+
+const makeNodeFsDb = ({
+  sqlite3,
+  fileName,
+  directory,
+  fs,
+}: {
+  sqlite3: WaSqlite.SQLiteAPI
+  fileName: string
+  directory: string
+  fs: FileSystem.FileSystem
+}) =>
+  Effect.gen(function* () {
+    // NOTE to keep the filePath short, we use the directory name in the vfs name
+    // If this is becoming a problem, we can use a hashed version of the directory name
+    const vfsName = `node-fs-${directory}`
+    if (nodeFsVfs === undefined) {
+      // TODO refactor with Effect FileSystem instead of using `node:fs` directly inside of NodeFS
+      nodeFsVfs = new NodeFS(vfsName, (sqlite3 as any).module, directory)
+      // @ts-expect-error TODO fix types
+      sqlite3.vfs_register(nodeFsVfs, false)
+    }
+
+    yield* fs.makeDirectory(directory, { recursive: true })
+
+    const FILE_NAME_MAX_LENGTH = 56
+    if (fileName.length > FILE_NAME_MAX_LENGTH) {
+      throw new Error(`File name ${fileName} is too long. Maximum length is ${FILE_NAME_MAX_LENGTH} characters.`)
+    }
+
+    // NOTE SQLite will return a "disk I/O error" if the file path is too long.
+    const dbPointer = sqlite3.open_v2Sync(fileName, undefined, vfsName)
+
+    return { dbPointer, vfs: nodeFsVfs }
+  }).pipe(UnexpectedError.mapToUnexpectedError)

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo",
+  },
+  "include": ["./src"],
+  "references": [{ "path": "../common" }, { "path": "../utils" }]
+}