@livestore/sqlite-wasm 0.4.0-dev.3 → 0.4.0-dev.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livestore/sqlite-wasm",
-  "version": "0.4.0-dev.3",
+  "version": "0.4.0-dev.6",
   "type": "module",
   "exports": {
     ".": {
@@ -9,6 +9,7 @@
     },
     "./load-wasm": {
       "types": "./dist/load-wasm/mod.browser.d.ts",
+      "workerd": "./dist/load-wasm/mod.workerd.js",
       "browser": "./dist/load-wasm/mod.browser.js",
       "worker": "./dist/load-wasm/mod.browser.js",
       "node": "./dist/load-wasm/mod.node.js",
@@ -18,20 +19,28 @@
       "types": "./dist/node/mod.d.ts",
       "default": "./dist/node/mod.js"
     },
+    "./cf": {
+      "types": "./dist/cf/mod.d.ts",
+      "default": "./dist/cf/mod.js"
+    },
     "./browser": {
       "types": "./dist/browser/mod.d.ts",
       "default": "./dist/browser/mod.js"
     }
   },
   "dependencies": {
-    "@livestore/wa-sqlite": "1.0.8-dev.4",
-    "@livestore/common": "0.4.0-dev.3",
-    "@livestore/utils": "0.4.0-dev.3"
+    "@cloudflare/workers-types": "4.20250823.0",
+    "@livestore/common": "0.4.0-dev.6",
+    "@livestore/common-cf": "0.4.0-dev.6",
+    "@livestore/utils": "0.4.0-dev.6",
+    "@livestore/wa-sqlite": "0.4.0-dev.6"
   },
   "devDependencies": {
-    "@types/chrome": "^0.1.1",
+    "@types/chrome": "^0.1.4",
     "@types/node": "24.2.0",
-    "@types/wicg-file-system-access": "^2023.10.6"
+    "@types/wicg-file-system-access": "^2023.10.6",
+    "vitest": "3.2.4",
+    "wrangler": "^4.32.0"
   },
   "files": [
     "package.json",
@@ -42,5 +51,9 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {}
+  "scripts": {
+    "test": "vitest",
+    "test:ui": "vitest --ui",
+    "test:watch": "vitest --watch"
+  }
 }

package/src/browser/mod.ts

@@ -1,5 +1,6 @@
 import type { MakeSqliteDb, PersistenceInfo, SqliteDb } from '@livestore/common'
 import { Effect, Hash } from '@livestore/utils/effect'
+import type { SQLiteAPI } from '@livestore/wa-sqlite'
 import type { MemoryVFS } from '@livestore/wa-sqlite/src/examples/MemoryVFS.js'
 
 import { makeInMemoryDb } from '../in-memory-vfs.ts'

package/src/browser/opfs/AccessHandlePoolVFS.ts

@@ -20,7 +20,33 @@ const HEADER_OFFSET_DATA = SECTOR_SIZE
 const PERSISTENT_FILE_TYPES =
   VFS.SQLITE_OPEN_MAIN_DB | VFS.SQLITE_OPEN_MAIN_JOURNAL | VFS.SQLITE_OPEN_SUPER_JOURNAL | VFS.SQLITE_OPEN_WAL
 
-const DEFAULT_CAPACITY = 6
+// OPFS file pool capacity must be predicted rather than dynamically increased because
+// capacity expansion (addCapacity) is async while SQLite operations are synchronous.
+// We cannot await in the middle of sqlite3.step() calls without making the API async.
+//
+// We over-allocate because:
+// 1. SQLite’s temporary file usage is not part of its API contract.
+//    Future SQLite versions may create additional temporary files without notice.
+//    See: https://www.sqlite.org/tempfiles.html
+// 2. In the future, we may change how we operate the SQLite DBs,
+//    which may increase the number of files needed.
+//    e.g. enabling the WAL mode, using multi-DB transactions, etc.
+//
+// TRADEOFF: Higher capacity means the VFS opens and keeps more file handles, consuming
+// browser resources. Lower capacity risks "SQLITE_CANTOPEN" errors during operations.
+//
+// CAPACITY CALCULATION:
+// - 2 main databases (state + eventlog) × 4 files each (main, journal, WAL, shm) = 8 files
+// - Up to 5 SQLite temporary files (super-journal, temp DB, materializations,
+//   transient indices, VACUUM temp DB) = 5 files
+// - 3 old state databases after migrations for debugging during development = 3 files
+// - Safety buffer for future SQLite versions and unpredictable usage = 4 files
+// Total: 20 files
+//
+// References:
+// - https://sqlite.org/forum/info/a3da1e34d8
+// - https://www.sqlite.org/tempfiles.html
+const DEFAULT_CAPACITY = 20
 
 /**
  * This VFS uses the updated Access Handle API with all synchronous methods
@@ -215,6 +241,16 @@ export class AccessHandlePoolVFS extends FacadeVFS {
     return this.#mapAccessHandleToName.size
   }
 
+  /**
+   * Get all currently tracked SQLite file paths.
+   * This can be used by higher-level components for file management operations.
+   *
+   * @returns Array of currently active SQLite file paths
+   */
+  getTrackedFilePaths(): string[] {
+    return Array.from(this.#mapPathToAccessHandle.keys())
+  }
+
   /**
    * Increase the capacity of the file system by n.
    */

package/src/cf/BlockManager.ts

@@ -0,0 +1,225 @@
+import type { CfTypes } from '@livestore/common-cf'
+
+export interface BlockRange {
+  startBlock: number
+  endBlock: number
+  startOffset: number
+  endOffset: number
+}
+
+export interface BlockData {
+  blockId: number
+  data: Uint8Array
+}
+
+/**
+ * BlockManager handles the conversion between file operations and block-based storage
+ * for the CloudflareSqlVFS. It manages fixed-size blocks stored in SQL tables.
+ */
+export class BlockManager {
+  private readonly blockSize: number
+
+  constructor(blockSize: number = 64 * 1024) {
+    this.blockSize = blockSize
+  }
+
+  /**
+   * Calculate which blocks are needed for a given file operation
+   */
+  calculateBlockRange(offset: number, length: number): BlockRange {
+    const startBlock = Math.floor(offset / this.blockSize)
+    const endBlock = Math.floor((offset + length - 1) / this.blockSize)
+    const startOffset = offset % this.blockSize
+    const endOffset = ((offset + length - 1) % this.blockSize) + 1
+
+    return {
+      startBlock,
+      endBlock,
+      startOffset,
+      endOffset,
+    }
+  }
+
+  /**
+   * Read blocks from SQL storage and return as a Map
+   */
+  readBlocks(sql: CfTypes.SqlStorage, filePath: string, blockIds: number[]): Map<number, Uint8Array> {
+    const blocks = new Map<number, Uint8Array>()
+
+    if (blockIds.length === 0) {
+      return blocks
+    }
+
+    // Build IN clause for efficient querying
+    const placeholders = blockIds.map(() => '?').join(',')
+    const query = `
+      SELECT block_id, block_data 
+      FROM vfs_blocks 
+      WHERE file_path = ? AND block_id IN (${placeholders})
+      ORDER BY block_id
+    `
+
+    const cursor = sql.exec<{ block_id: number; block_data: ArrayBuffer }>(query, filePath, ...blockIds)
+
+    for (const row of cursor) {
+      blocks.set(row.block_id, new Uint8Array(row.block_data))
+    }
+
+    return blocks
+  }
+
+  /**
+   * Write blocks to SQL storage using exec for now (prepared statements later)
+   */
+  writeBlocks(sql: CfTypes.SqlStorage, filePath: string, blocks: Map<number, Uint8Array>): void {
+    if (blocks.size === 0) {
+      return
+    }
+
+    for (const [blockId, data] of blocks) {
+      sql.exec(
+        'INSERT OR REPLACE INTO vfs_blocks (file_path, block_id, block_data) VALUES (?, ?, ?)',
+        filePath,
+        blockId,
+        data,
+      )
+    }
+  }
+
+  /**
+   * Delete blocks at or after the specified block ID (used for truncation)
+   */
+  deleteBlocksAfter(sql: CfTypes.SqlStorage, filePath: string, startBlockId: number): void {
+    sql.exec('DELETE FROM vfs_blocks WHERE file_path = ? AND block_id >= ?', filePath, startBlockId)
+  }
+
+  /**
+   * Split write data into blocks, handling partial blocks at boundaries
+   */
+  splitIntoBlocks(
+    data: Uint8Array,
+    offset: number,
+  ): Map<number, { blockId: number; blockOffset: number; data: Uint8Array }> {
+    const blocks = new Map<number, { blockId: number; blockOffset: number; data: Uint8Array }>()
+
+    let remainingData = data
+    let currentOffset = offset
+
+    while (remainingData.length > 0) {
+      const blockId = Math.floor(currentOffset / this.blockSize)
+      const blockOffset = currentOffset % this.blockSize
+      const bytesToWrite = Math.min(remainingData.length, this.blockSize - blockOffset)
+
+      const blockData = remainingData.slice(0, bytesToWrite)
+      blocks.set(blockId, {
+        blockId,
+        blockOffset,
+        data: blockData,
+      })
+
+      remainingData = remainingData.slice(bytesToWrite)
+      currentOffset += bytesToWrite
+    }
+
+    return blocks
+  }
+
+  /**
+   * Assemble read data from blocks into a continuous buffer
+   */
+  assembleBlocks(blocks: Map<number, Uint8Array>, range: BlockRange, requestedLength: number): Uint8Array {
+    const result = new Uint8Array(requestedLength)
+    let resultOffset = 0
+
+    for (let blockId = range.startBlock; blockId <= range.endBlock; blockId++) {
+      const blockData = blocks.get(blockId)
+      if (!blockData) {
+        // Block not found - fill with zeros (sparse file behavior)
+        const zeroLength = Math.min(this.blockSize, requestedLength - resultOffset)
+        // result is already zero-filled by default
+        resultOffset += zeroLength
+        continue
+      }
+
+      // Calculate the slice of this block we need
+      const blockStartOffset = blockId === range.startBlock ? range.startOffset : 0
+      const blockEndOffset = blockId === range.endBlock ? range.endOffset : blockData.length
+      const sliceLength = blockEndOffset - blockStartOffset
+
+      if (sliceLength > 0) {
+        const slice = blockData.slice(blockStartOffset, blockEndOffset)
+        result.set(slice, resultOffset)
+        resultOffset += sliceLength
+      }
+    }
+
+    return result
+  }
+
+  /**
+   * Handle partial block writes by reading existing block, modifying, and returning complete block
+   */
+  mergePartialBlock(
+    sql: CfTypes.SqlStorage,
+    filePath: string,
+    blockId: number,
+    blockOffset: number,
+    newData: Uint8Array,
+  ): Uint8Array {
+    // Read existing block data if it exists
+    const existingBlocks = this.readBlocks(sql, filePath, [blockId])
+    const existingBlock = existingBlocks.get(blockId) || new Uint8Array(this.blockSize)
+
+    // Create a new block with the merged data
+    const mergedBlock = new Uint8Array(this.blockSize)
+    mergedBlock.set(existingBlock)
+    mergedBlock.set(newData, blockOffset)
+
+    return mergedBlock
+  }
+
+  /**
+   * Get statistics about block usage for a file
+   */
+  getBlockStats(
+    sql: CfTypes.SqlStorage,
+    filePath: string,
+  ): { totalBlocks: number; storedBlocks: number; totalBytes: number } {
+    const blockStatsCursor = sql.exec<{ stored_blocks: number; total_bytes: number }>(
+      `SELECT 
+        COUNT(*) as stored_blocks,
+        COALESCE(SUM(LENGTH(block_data)), 0) as total_bytes
+      FROM vfs_blocks 
+      WHERE file_path = ?`,
+      filePath,
+    )
+
+    const result = blockStatsCursor.one()
+
+    // Get file size to calculate theoretical total blocks
+    const fileSizeCursor = sql.exec<{ file_size: number }>(
+      'SELECT file_size FROM vfs_files WHERE file_path = ?',
+      filePath,
+    )
+
+    let fileSize = 0
+    try {
+      const fileSizeResult = fileSizeCursor.one()
+      fileSize = fileSizeResult.file_size
+    } catch {
+      // File doesn't exist
+    }
+
+    const totalBlocks = Math.ceil(fileSize / this.blockSize)
+
+    return {
+      totalBlocks,
+      storedBlocks: result.stored_blocks,
+      totalBytes: result.total_bytes,
+    }
+  }
+
+  getBlockSize(): number {
+    return this.blockSize
+  }
+}