@livestore/sqlite-wasm 0.4.0-dev.2 → 0.4.0-dev.20

package/src/browser/opfs/index.ts

@@ -1,4 +1,5 @@
-import { Effect } from '@livestore/utils/effect'
+import { Effect, type Scope } from '@livestore/utils/effect'
+import type { Opfs } from '@livestore/utils/effect/browser'
 import type * as WaSqlite from '@livestore/wa-sqlite'
 
 import { AccessHandlePoolVFS } from './AccessHandlePoolVFS.ts'
@@ -14,7 +15,7 @@ export const makeOpfsDb = ({
   sqlite3: WaSqlite.SQLiteAPI
   directory: string
   fileName: string
-}) =>
+}): Effect.Effect<{ dbPointer: number; vfs: AccessHandlePoolVFS }, never, Opfs.Opfs | Scope.Scope> =>
   Effect.gen(function* () {
     // Replace all special characters with underscores
     const safePath = directory.replaceAll(/["*/:<>?\\|]/g, '_')
@@ -22,7 +23,7 @@ export const makeOpfsDb = ({
     const vfsName = `opfs${pathSegment}`
 
     if (sqlite3.vfs_registered.has(vfsName) === false) {
-      const vfs = yield* Effect.promise(() => AccessHandlePoolVFS.create(vfsName, directory, (sqlite3 as any).module))
+      const vfs = yield* AccessHandlePoolVFS.create(vfsName, directory, (sqlite3 as any).module)
 
       sqlite3.vfs_register(vfs, false)
       opfsVfsMap.set(vfsName, vfs)

package/src/browser/opfs/opfs-sah-pool.ts

@@ -14,7 +14,7 @@ const PERSISTENT_FILE_TYPES =
 
 const textDecoder = new TextDecoder()
 
-export const decodeSAHPoolFilename = async (file: File): Promise<string> => {
+export const decodeAccessHandlePoolFilename = async (file: File): Promise<string> => {
   // Read the path and digest of the path from the file.
   const corpus = new Uint8Array(await file.slice(0, HEADER_CORPUS_SIZE).arrayBuffer())
 

package/src/cf/BlockManager.ts

@@ -0,0 +1,225 @@
+import type { CfTypes } from '@livestore/common-cf'
+
+export interface BlockRange {
+  startBlock: number
+  endBlock: number
+  startOffset: number
+  endOffset: number
+}
+
+export interface BlockData {
+  blockId: number
+  data: Uint8Array
+}
+
+/**
+ * BlockManager handles the conversion between file operations and block-based storage
+ * for the CloudflareSqlVFS. It manages fixed-size blocks stored in SQL tables.
+ */
+export class BlockManager {
+  private readonly blockSize: number
+
+  constructor(blockSize: number = 64 * 1024) {
+    this.blockSize = blockSize
+  }
+
+  /**
+   * Calculate which blocks are needed for a given file operation
+   */
+  calculateBlockRange(offset: number, length: number): BlockRange {
+    const startBlock = Math.floor(offset / this.blockSize)
+    const endBlock = Math.floor((offset + length - 1) / this.blockSize)
+    const startOffset = offset % this.blockSize
+    const endOffset = ((offset + length - 1) % this.blockSize) + 1
+
+    return {
+      startBlock,
+      endBlock,
+      startOffset,
+      endOffset,
+    }
+  }
+
+  /**
+   * Read blocks from SQL storage and return as a Map
+   */
+  readBlocks(sql: CfTypes.SqlStorage, filePath: string, blockIds: number[]): Map<number, Uint8Array> {
+    const blocks = new Map<number, Uint8Array>()
+
+    if (blockIds.length === 0) {
+      return blocks
+    }
+
+    // Build IN clause for efficient querying
+    const placeholders = blockIds.map(() => '?').join(',')
+    const query = `
+      SELECT block_id, block_data 
+      FROM vfs_blocks 
+      WHERE file_path = ? AND block_id IN (${placeholders})
+      ORDER BY block_id
+    `
+
+    const cursor = sql.exec<{ block_id: number; block_data: ArrayBuffer }>(query, filePath, ...blockIds)
+
+    for (const row of cursor) {
+      blocks.set(row.block_id, new Uint8Array(row.block_data))
+    }
+
+    return blocks
+  }
+
+  /**
+   * Write blocks to SQL storage using exec for now (prepared statements later)
+   */
+  writeBlocks(sql: CfTypes.SqlStorage, filePath: string, blocks: Map<number, Uint8Array>): void {
+    if (blocks.size === 0) {
+      return
+    }
+
+    for (const [blockId, data] of blocks) {
+      sql.exec(
+        'INSERT OR REPLACE INTO vfs_blocks (file_path, block_id, block_data) VALUES (?, ?, ?)',
+        filePath,
+        blockId,
+        data,
+      )
+    }
+  }
+
+  /**
+   * Delete blocks at or after the specified block ID (used for truncation)
+   */
+  deleteBlocksAfter(sql: CfTypes.SqlStorage, filePath: string, startBlockId: number): void {
+    sql.exec('DELETE FROM vfs_blocks WHERE file_path = ? AND block_id >= ?', filePath, startBlockId)
+  }
+
+  /**
+   * Split write data into blocks, handling partial blocks at boundaries
+   */
+  splitIntoBlocks(
+    data: Uint8Array,
+    offset: number,
+  ): Map<number, { blockId: number; blockOffset: number; data: Uint8Array }> {
+    const blocks = new Map<number, { blockId: number; blockOffset: number; data: Uint8Array }>()
+
+    let remainingData = data
+    let currentOffset = offset
+
+    while (remainingData.length > 0) {
+      const blockId = Math.floor(currentOffset / this.blockSize)
+      const blockOffset = currentOffset % this.blockSize
+      const bytesToWrite = Math.min(remainingData.length, this.blockSize - blockOffset)
+
+      const blockData = remainingData.slice(0, bytesToWrite)
+      blocks.set(blockId, {
+        blockId,
+        blockOffset,
+        data: blockData,
+      })
+
+      remainingData = remainingData.slice(bytesToWrite)
+      currentOffset += bytesToWrite
+    }
+
+    return blocks
+  }
+
+  /**
+   * Assemble read data from blocks into a continuous buffer
+   */
+  assembleBlocks(blocks: Map<number, Uint8Array>, range: BlockRange, requestedLength: number): Uint8Array {
+    const result = new Uint8Array(requestedLength)
+    let resultOffset = 0
+
+    for (let blockId = range.startBlock; blockId <= range.endBlock; blockId++) {
+      const blockData = blocks.get(blockId)
+      if (!blockData) {
+        // Block not found - fill with zeros (sparse file behavior)
+        const zeroLength = Math.min(this.blockSize, requestedLength - resultOffset)
+        // result is already zero-filled by default
+        resultOffset += zeroLength
+        continue
+      }
+
+      // Calculate the slice of this block we need
+      const blockStartOffset = blockId === range.startBlock ? range.startOffset : 0
+      const blockEndOffset = blockId === range.endBlock ? range.endOffset : blockData.length
+      const sliceLength = blockEndOffset - blockStartOffset
+
+      if (sliceLength > 0) {
+        const slice = blockData.slice(blockStartOffset, blockEndOffset)
+        result.set(slice, resultOffset)
+        resultOffset += sliceLength
+      }
+    }
+
+    return result
+  }
+
+  /**
+   * Handle partial block writes by reading existing block, modifying, and returning complete block
+   */
+  mergePartialBlock(
+    sql: CfTypes.SqlStorage,
+    filePath: string,
+    blockId: number,
+    blockOffset: number,
+    newData: Uint8Array,
+  ): Uint8Array {
+    // Read existing block data if it exists
+    const existingBlocks = this.readBlocks(sql, filePath, [blockId])
+    const existingBlock = existingBlocks.get(blockId) || new Uint8Array(this.blockSize)
+
+    // Create a new block with the merged data
+    const mergedBlock = new Uint8Array(this.blockSize)
+    mergedBlock.set(existingBlock)
+    mergedBlock.set(newData, blockOffset)
+
+    return mergedBlock
+  }
+
+  /**
+   * Get statistics about block usage for a file
+   */
+  getBlockStats(
+    sql: CfTypes.SqlStorage,
+    filePath: string,
+  ): { totalBlocks: number; storedBlocks: number; totalBytes: number } {
+    const blockStatsCursor = sql.exec<{ stored_blocks: number; total_bytes: number }>(
+      `SELECT 
+        COUNT(*) as stored_blocks,
+        COALESCE(SUM(LENGTH(block_data)), 0) as total_bytes
+      FROM vfs_blocks 
+      WHERE file_path = ?`,
+      filePath,
+    )
+
+    const result = blockStatsCursor.one()
+
+    // Get file size to calculate theoretical total blocks
+    const fileSizeCursor = sql.exec<{ file_size: number }>(
+      'SELECT file_size FROM vfs_files WHERE file_path = ?',
+      filePath,
+    )
+
+    let fileSize = 0
+    try {
+      const fileSizeResult = fileSizeCursor.one()
+      fileSize = fileSizeResult.file_size
+    } catch {
+      // File doesn't exist
+    }
+
+    const totalBlocks = Math.ceil(fileSize / this.blockSize)
+
+    return {
+      totalBlocks,
+      storedBlocks: result.stored_blocks,
+      totalBytes: result.total_bytes,
+    }
+  }
+
+  getBlockSize(): number {
+    return this.blockSize
+  }
+}

package/src/cf/CloudflareSqlVFS.ts

@@ -0,0 +1,450 @@
+import type { CfTypes } from '@livestore/common-cf'
+import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
+import { FacadeVFS } from '../FacadeVFS.ts'
+import { BlockManager } from './BlockManager.ts'
+
+const SECTOR_SIZE = 4096
+
+// Block size for SQL-based storage (same as CloudflareWorkerVFS for consistency)
+const BLOCK_SIZE = 64 * 1024 // 64 KiB
+
+// Maximum number of open files
+const DEFAULT_MAX_FILES = 100
+
+// These file types are expected to persist in the file system
+const PERSISTENT_FILE_TYPES =
+  VFS.SQLITE_OPEN_MAIN_DB | VFS.SQLITE_OPEN_MAIN_JOURNAL | VFS.SQLITE_OPEN_SUPER_JOURNAL | VFS.SQLITE_OPEN_WAL
+
+interface FileMetadata {
+  path: string
+  size: number
+  flags: number
+  created: number
+  modified: number
+}
+
+interface FileHandle {
+  path: string
+  flags: number
+  metadata: FileMetadata
+}
+
+export interface SqlVfsOptions {
+  maxFiles?: number
+  blockSize?: number
+}
+
+/**
+ * VFS implementation using Cloudflare Durable Object SQL storage as the backend.
+ * This provides a synchronous VFS interface by leveraging SQL's synchronous API.
+ *
+ * Storage Strategy:
+ * - Files are stored as blocks in SQL tables for efficient I/O
+ * - File metadata stored in vfs_files table
+ * - File data stored as fixed-size blocks in vfs_blocks table
+ * - Synchronous operations via SQL's synchronous API
+ *
+ * Key advantages over async VFS:
+ * - No async/await complexity
+ * - Native SQL ACID properties
+ * - Efficient range queries for file operations
+ * - Built-in consistency and durability
+ */
+export class CloudflareSqlVFS extends FacadeVFS {
+  log = null
+
+  #sql: CfTypes.SqlStorage
+  #initialized = false
+  #blockManager: BlockManager
+
+  // File management
+  #openFiles = new Map<number, FileHandle>()
+  #maxFiles: number
+
+  static async create(name: string, sql: CfTypes.SqlStorage, module: any, options: SqlVfsOptions = {}) {
+    const vfs = new CloudflareSqlVFS(name, sql, module, options)
+    await vfs.isReady()
+    return vfs
+  }
+
+  constructor(name: string, sql: CfTypes.SqlStorage, module: any, options: SqlVfsOptions = {}) {
+    super(name, module)
+    this.#sql = sql
+    this.#maxFiles = options.maxFiles || DEFAULT_MAX_FILES
+    this.#blockManager = new BlockManager(options.blockSize || BLOCK_SIZE)
+  }
+
+  /**
+   * Initialize the VFS by setting up SQL schema
+   */
+  async isReady(): Promise<boolean> {
+    if (this.#initialized) {
+      return true
+    }
+
+    try {
+      // Initialize SQL schema
+      this.#initializeSchema()
+
+      // Clean up non-persistent files from previous sessions
+      this.#cleanupNonPersistentFiles()
+
+      this.#initialized = true
+      return true
+    } catch (error) {
+      console.error('CloudflareSqlVFS initialization failed:', error)
+      return false
+    }
+  }
+
+  /**
+   * Initialize the SQL schema for the VFS
+   */
+  #initializeSchema(): void {
+    // Execute each statement individually to avoid parsing issues
+    const statements = [
+      `CREATE TABLE IF NOT EXISTS vfs_files (
+        file_path TEXT PRIMARY KEY,
+        file_size INTEGER NOT NULL DEFAULT 0,
+        flags INTEGER NOT NULL DEFAULT 0,
+        created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+        modified_at INTEGER NOT NULL DEFAULT (unixepoch())
+      )`,
+
+      `CREATE TABLE IF NOT EXISTS vfs_blocks (
+        file_path TEXT NOT NULL,
+        block_id INTEGER NOT NULL,
+        block_data BLOB NOT NULL,
+        created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+        PRIMARY KEY (file_path, block_id),
+        FOREIGN KEY (file_path) REFERENCES vfs_files(file_path) ON DELETE CASCADE
+      )`,
+
+      `CREATE INDEX IF NOT EXISTS idx_vfs_blocks_range ON vfs_blocks(file_path, block_id)`,
+
+      `CREATE INDEX IF NOT EXISTS idx_vfs_files_modified ON vfs_files(modified_at)`,
+
+      `CREATE TRIGGER IF NOT EXISTS trg_vfs_files_update_modified 
+        AFTER UPDATE OF file_size ON vfs_files
+        BEGIN
+          UPDATE vfs_files SET modified_at = unixepoch() WHERE file_path = NEW.file_path;
+        END`,
+    ]
+
+    for (const statement of statements) {
+      try {
+        this.#sql.exec(statement)
+      } catch (error) {
+        console.error('Failed to execute schema statement:', statement)
+        throw error
+      }
+    }
+  }
+
+  /**
+   * Clean up non-persistent files from previous sessions
+   */
+  #cleanupNonPersistentFiles(): void {
+    try {
+      const cursor = this.#sql.exec<{ file_path: string; flags: number }>('SELECT file_path, flags FROM vfs_files')
+
+      const filesToDelete: string[] = []
+
+      for (const row of cursor) {
+        // Check if file should be persistent
+        if (!(row.flags & PERSISTENT_FILE_TYPES)) {
+          filesToDelete.push(row.file_path)
+        }
+      }
+
+      // Delete non-persistent files
+      for (const filePath of filesToDelete) {
+        this.#sql.exec('DELETE FROM vfs_files WHERE file_path = ?', filePath)
+      }
+    } catch (error) {
+      console.warn('Error during cleanup:', error)
+    }
+  }
+
+  // VFS Interface Implementation
+
+  jOpen(path: string, fileId: number, flags: number, pOutFlags: DataView): number {
+    try {
+      if (this.#openFiles.size >= this.#maxFiles) {
+        return VFS.SQLITE_CANTOPEN
+      }
+
+      // Check if file exists
+      const existingFile = this.#getFileMetadata(path)
+
+      if (!existingFile && !(flags & VFS.SQLITE_OPEN_CREATE)) {
+        return VFS.SQLITE_CANTOPEN
+      }
+
+      let metadata: FileMetadata
+
+      if (existingFile) {
+        metadata = existingFile
+      } else {
+        // Create new file
+        const now = Math.floor(Date.now() / 1000)
+        metadata = {
+          path,
+          size: 0,
+          flags,
+          created: now,
+          modified: now,
+        }
+
+        this.#sql.exec(
+          'INSERT INTO vfs_files (file_path, file_size, flags, created_at, modified_at) VALUES (?, ?, ?, ?, ?)',
+          path,
+          0,
+          flags,
+          now,
+          now,
+        )
+      }
+
+      // Store file handle
+      this.#openFiles.set(fileId, {
+        path,
+        flags,
+        metadata,
+      })
+
+      pOutFlags.setInt32(0, flags, true)
+      return VFS.SQLITE_OK
+    } catch (error) {
+      console.error('jOpen error:', error)
+      return VFS.SQLITE_CANTOPEN
+    }
+  }
+
+  jClose(fileId: number): number {
+    this.#openFiles.delete(fileId)
+    return VFS.SQLITE_OK
+  }
+
+  jRead(fileId: number, buffer: Uint8Array, offset: number): number {
+    try {
+      const handle = this.#openFiles.get(fileId)
+      if (!handle) {
+        return VFS.SQLITE_IOERR
+      }
+
+      const range = this.#blockManager.calculateBlockRange(offset, buffer.length)
+      const blockIds = []
+      for (let i = range.startBlock; i <= range.endBlock; i++) {
+        blockIds.push(i)
+      }
+
+      const blocks = this.#blockManager.readBlocks(this.#sql, handle.path, blockIds)
+      const data = this.#blockManager.assembleBlocks(blocks, range, buffer.length)
+
+      buffer.set(data)
+      return VFS.SQLITE_OK
+    } catch (error) {
+      console.error('jRead error:', error)
+      return VFS.SQLITE_IOERR
+    }
+  }
+
+  jWrite(fileId: number, data: Uint8Array, offset: number): number {
+    try {
+      const handle = this.#openFiles.get(fileId)
+      if (!handle) {
+        return VFS.SQLITE_IOERR
+      }
+
+      // Split write data into blocks
+      const writeBlocks = this.#blockManager.splitIntoBlocks(data, offset)
+      const finalBlocks = new Map<number, Uint8Array>()
+
+      for (const [blockId, blockInfo] of writeBlocks) {
+        let blockData: Uint8Array
+
+        if (blockInfo.blockOffset === 0 && blockInfo.data.length === this.#blockManager.getBlockSize()) {
+          // Full block write
+          blockData = blockInfo.data
+        } else {
+          // Partial block write - merge with existing data
+          blockData = this.#blockManager.mergePartialBlock(
+            this.#sql,
+            handle.path,
+            blockInfo.blockId,
+            blockInfo.blockOffset,
+            blockInfo.data,
+          )
+        }
+
+        finalBlocks.set(blockId, blockData)
+      }
+
+      // Write blocks to SQL storage
+      this.#blockManager.writeBlocks(this.#sql, handle.path, finalBlocks)
+
+      // Update file size if necessary
+      const newSize = Math.max(handle.metadata.size, offset + data.length)
+      if (newSize !== handle.metadata.size) {
+        this.#sql.exec('UPDATE vfs_files SET file_size = ? WHERE file_path = ?', newSize, handle.path)
+        handle.metadata.size = newSize
+      }
+
+      return VFS.SQLITE_OK
+    } catch (error) {
+      console.error('jWrite error:', error)
+      return VFS.SQLITE_IOERR
+    }
+  }
+
+  jTruncate(fileId: number, size: number): number {
+    try {
+      const handle = this.#openFiles.get(fileId)
+      if (!handle) {
+        return VFS.SQLITE_IOERR
+      }
+
+      // Calculate which block contains the new end of file
+      const lastBlockId = Math.floor(size / this.#blockManager.getBlockSize())
+
+      // Delete blocks beyond the truncation point
+      this.#blockManager.deleteBlocksAfter(this.#sql, handle.path, lastBlockId + 1)
+
+      // If truncating within a block, we need to truncate that block's data
+      if (size % this.#blockManager.getBlockSize() !== 0) {
+        const existingBlocks = this.#blockManager.readBlocks(this.#sql, handle.path, [lastBlockId])
+        const blockData = existingBlocks.get(lastBlockId)
+
+        if (blockData) {
+          const truncatedBlock = blockData.slice(0, size % this.#blockManager.getBlockSize())
+          const paddedBlock = new Uint8Array(this.#blockManager.getBlockSize())
+          paddedBlock.set(truncatedBlock)
+
+          const blocksToWrite = new Map([[lastBlockId, paddedBlock]])
+          this.#blockManager.writeBlocks(this.#sql, handle.path, blocksToWrite)
+        }
+      }
+
+      // Update file metadata
+      this.#sql.exec('UPDATE vfs_files SET file_size = ? WHERE file_path = ?', size, handle.path)
+      handle.metadata.size = size
+
+      return VFS.SQLITE_OK
+    } catch (error) {
+      console.error('jTruncate error:', error)
+      return VFS.SQLITE_IOERR
+    }
+  }
+
+  jSync(fileId: number, _flags: number): number {
+    // SQL storage provides immediate durability, so sync is effectively a no-op
+    const handle = this.#openFiles.get(fileId)
+    if (!handle) {
+      return VFS.SQLITE_IOERR
+    }
+    return VFS.SQLITE_OK
+  }
+
+  jFileSize(fileId: number, pSize64: DataView): number {
+    try {
+      const handle = this.#openFiles.get(fileId)
+      if (!handle) {
+        return VFS.SQLITE_IOERR
+      }
+
+      pSize64.setBigInt64(0, BigInt(handle.metadata.size), true)
+      return VFS.SQLITE_OK
+    } catch (error) {
+      console.error('jFileSize error:', error)
+      return VFS.SQLITE_IOERR
+    }
+  }
+
+  jDelete(path: string, _syncDir: number): number {
+    try {
+      this.#sql.exec('DELETE FROM vfs_files WHERE file_path = ?', path)
+      return VFS.SQLITE_OK
+    } catch (error) {
+      console.error('jDelete error:', error)
+      return VFS.SQLITE_IOERR
+    }
+  }
+
+  jAccess(path: string, _flags: number, pResOut: DataView): number {
+    try {
+      const metadata = this.#getFileMetadata(path)
+      pResOut.setInt32(0, metadata ? 1 : 0, true)
+      return VFS.SQLITE_OK
+    } catch (error) {
+      console.error('jAccess error:', error)
+      return VFS.SQLITE_IOERR
+    }
+  }
+
+  jSectorSize(_fileId: number): number {
+    return SECTOR_SIZE
+  }
+
+  jDeviceCharacteristics(_fileId: number): number {
+    return VFS.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN
+  }
+
+  // Helper methods
+
+  #getFileMetadata(path: string): FileMetadata | undefined {
+    try {
+      const cursor = this.#sql.exec<{
+        file_path: string
+        file_size: number
+        flags: number
+        created_at: number
+        modified_at: number
+      }>('SELECT file_path, file_size, flags, created_at, modified_at FROM vfs_files WHERE file_path = ?', path)
+
+      const row = cursor.one()
+      return {
+        path: row.file_path,
+        size: row.file_size,
+        flags: row.flags,
+        created: row.created_at,
+        modified: row.modified_at,
+      }
+    } catch {
+      return undefined
+    }
+  }
+
+  // Statistics and debugging
+
+  getStats(): {
+    activeFiles: number
+    openFiles: number
+    maxFiles: number
+    blockSize: number
+    totalStoredBytes: number
+  } {
+    try {
+      const cursor = this.#sql.exec<{ total_files: number; total_bytes: number }>(
+        'SELECT COUNT(*) as total_files, COALESCE(SUM(LENGTH(block_data)), 0) as total_bytes FROM vfs_files LEFT JOIN vfs_blocks USING (file_path)',
+      )
+      const stats = cursor.one()
+
+      return {
+        activeFiles: stats.total_files,
+        openFiles: this.#openFiles.size,
+        maxFiles: this.#maxFiles,
+        blockSize: this.#blockManager.getBlockSize(),
+        totalStoredBytes: stats.total_bytes,
+      }
+    } catch {
+      return {
+        activeFiles: 0,
+        openFiles: this.#openFiles.size,
+        maxFiles: this.#maxFiles,
+        blockSize: this.#blockManager.getBlockSize(),
+        totalStoredBytes: 0,
+      }
+    }
+  }
+}