@livestore/sqlite-wasm 0.4.0-dev.9 → 0.5.0-dev.0

package/src/cf/mod.ts

@@ -1,16 +1,20 @@
 // import path from 'node:path'
 
-import { type MakeSqliteDb, type PersistenceInfo, type SqliteDb, UnexpectedError } from '@livestore/common'
+import { type MakeSqliteDb, type PersistenceInfo, type SqliteDb, UnknownError } from '@livestore/common'
 import type { CfTypes } from '@livestore/common-cf'
 import { Effect } from '@livestore/utils/effect'
 import type * as WaSqlite from '@livestore/wa-sqlite'
 import type { MemoryVFS } from '@livestore/wa-sqlite/src/examples/MemoryVFS.js'
+
 import { makeInMemoryDb } from '../in-memory-vfs.ts'
 import { makeSqliteDb } from '../make-sqlite-db.ts'
-import { CloudflareSqlVFS } from './CloudflareSqlVFS.ts'
+import { CloudflareDurableObjectVFS } from './CloudflareDurableObjectVFS.ts'
 
-export { BlockManager } from './BlockManager.ts'
-export { CloudflareSqlVFS } from './CloudflareSqlVFS.ts'
+export {
+  CloudflareDurableObjectVFS,
+  PAGE_SIZE as CF_SQL_VFS_PAGE_SIZE,
+  REQUIRED_PRAGMAS as CF_SQL_VFS_REQUIRED_PRAGMAS,
+} from './CloudflareDurableObjectVFS.ts'
 export { CloudflareWorkerVFS } from './CloudflareWorkerVFS.ts'
 
 export type CloudflareDatabaseMetadataInMemory = {
@@ -24,8 +28,7 @@ export type CloudflareDatabaseMetadataInMemory = {
 
 export type CloudflareDatabaseMetadataFs = {
   _tag: 'storage'
-  // vfs: CloudflareWorkerVFS
-  vfs: CloudflareSqlVFS
+  vfs: CloudflareDurableObjectVFS
   dbPointer: number
   persistenceInfo: PersistenceInfo
   deleteDb: () => void
@@ -115,18 +118,10 @@ const makeCloudflareFsDb = ({
     // If this is becoming a problem, we can use a hashed version of the directory name
     const vfsName = `cf-do-sqlite-${fileName}`
     if (sqlite3.vfs_registered.has(vfsName) === false) {
-      // TODO refactor with Effect FileSystem instead of using `node:fs` directly inside of CloudflareWorkerVFS
-      // const nodeFsVfs = new CloudflareWorkerVFS(vfsName, storage, (sqlite3 as any).module)
-      const nodeFsVfs = new CloudflareSqlVFS(vfsName, storage.sql, (sqlite3 as any).module)
-
-      // Initialize the VFS schema before registering it
-      const isReady = yield* Effect.promise(() => nodeFsVfs.isReady())
-      if (!isReady) {
-        throw new Error(`Failed to initialize CloudflareSqlVFS for ${vfsName}`)
-      }
+      const vfs = new CloudflareDurableObjectVFS(vfsName, storage.sql, (sqlite3 as any).module)
 
       // @ts-expect-error TODO fix types
-      sqlite3.vfs_register(nodeFsVfs, false)
+      sqlite3.vfs_register(vfs, false)
     }
 
     // yield* fs.makeDirectory(directory, { recursive: true })
@@ -140,4 +135,4 @@ const makeCloudflareFsDb = ({
     const dbPointer = sqlite3.open_v2Sync(fileName, undefined, vfsName)
 
     return { dbPointer, vfs: {} as UNUSED<'only needed in web adapter currently and should longer-term be removed'> }
-  }).pipe(UnexpectedError.mapToUnexpectedError)
+  }).pipe(UnknownError.mapToUnknownError)

package/src/cf/test/README.md

@@ -5,11 +5,13 @@ This directory contains comprehensive tests for both SQLite WASM VFS implementat
 ## Directory Structure
 
 ### `/sql/` - SQL Storage Tests
-Tests for the CloudflareSqlVFS implementation, which uses Cloudflare's DurableObject SQL API for storage.
+
+Tests for the CloudflareDurableObjectVFS implementation, which uses Cloudflare's [SQLite in DurableObjects](https://developers.cloudflare.com/durable-objects/api/sql-storage/) for storage.
 
 - **`cloudflare-sql-vfs-core.test.ts`** - Core SQL VFS functionality tests
 
-### `/async-storage/` - Async Storage Tests  
+### `/async-storage/` - Async Storage Tests
+
 Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorage (key-value) for file storage.
 
 - **`cloudflare-worker-vfs-core.test.ts`** - Basic VFS operations (open, read, write, close, sync)
@@ -20,19 +22,22 @@ Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorag
 ## Test Architecture
 
 ### Testing Framework
-- **Vitest** with **@cloudflare/vitest-pool-workers** for Workers runtime testing
-- **Isolated Storage** - Each test gets fresh storage instances
-- **Real Runtime** - Tests run in the actual Cloudflare Workers runtime environment
+
+- **Vitest** in the Node test environment
+- **Mocked Cloudflare storage interfaces** - Tests provide in-memory `DurableObjectStorage` and SQL storage doubles
+- **Per-test setup** - Each test creates fresh mock storage and VFS instances
 
 ## VFS Implementation Comparison
 
-### SQL Storage VFS (`CloudflareSqlVFS`)
-- **Backend**: Cloudflare DurableObject SQL API
-- **Storage Model**: Relational tables with blocks and metadata
-- **Advantages**: ACID transactions, complex queries, relational integrity
+### SQL Storage VFS (`CloudflareDurableObjectVFS`)
+
+- **Backend**: Cloudflare's SQLite in Durable Objects
+- **Storage Model**: Single `vfs_pages` table keyed by file path and page number
+- **Advantages**: Simple synchronous SQL storage with direct page lookups
 - **Use Case**: Applications requiring complex data relationships
 
-### Async Storage VFS (`CloudflareWorkerVFS`)  
+### Async Storage VFS (`CloudflareWorkerVFS`)
+
 - **Backend**: DurableObjectStorage (key-value)
 - **Storage Model**: 64 KiB chunks with LRU caching
 - **Advantages**: Simple API, optimized for large files, better caching
@@ -41,11 +46,14 @@ Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorag
 ## Key Design Decisions Tested
 
 ### SQL Storage Approach
-- **Block-based Storage**: Files stored as blocks in SQL tables
-- **Metadata Management**: File metadata in dedicated tables
-- **Transaction Safety**: ACID compliance for data integrity
+
+- **Page-based Storage**: SQLite pages stored directly in `vfs_pages`
+- **File-path Isolation**: Each logical SQLite file is scoped by `file_path`
+- **Handle Tracking**: Open SQLite file handles map back to persisted file paths
+- **Transaction Model**: Page writes are persisted directly through Durable Object SQL
 
 ### Async Storage Approach
+
 - **64 KiB Chunking Strategy**: Optimized for SQLite I/O patterns
 - **Synchronous Interface with Async Backend**: Aggressive caching + background sync
 - **Memory Management**: LRU cache for chunks, complete metadata cache
@@ -54,14 +62,11 @@ Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorag
 ## Running Tests
 
 ### Prerequisites
+
 1. Install dependencies:
-   ```bash
-   pnpm install
-   ```
 
-2. Ensure Wrangler configuration is correct:
    ```bash
-   # Check wrangler.toml exists and has correct Durable Object bindings
+   pnpm install
    ```
 
 ### Test Commands
@@ -90,30 +95,33 @@ pnpm test --coverage
 
 ### Test Environment
 
-- **Runtime**: Cloudflare Workers (via workerd)
-- **Storage**: Isolated per-test storage instances
+- **Runtime**: Node via Vitest
+- **Storage**: Mocked Durable Object storage and SQL storage instances created per test
+- **Scope**: Unit/integration coverage for VFS behavior against Cloudflare-shaped interfaces, not workerd runtime coverage
 
 ## Test Structure
 
 ### SQL Storage Test Pattern
+
 ```typescript
 import { describe, it, expect, beforeEach } from 'vitest'
-import { type Cf, CloudflareSqlVFS } from '../../mod.ts'
+import type { CfTypes } from '@livestore/common-cf'
+import { CloudflareDurableObjectVFS } from '../../mod.ts'
 
 describe('SQL VFS Test Suite', () => {
-  let vfs: CloudflareSqlVFS
-  let mockSql: Cf.SqlStorage
+  let vfs: CloudflareDurableObjectVFS
+  let mockSql: CfTypes.SqlStorage
   let queryLog: string[]
 
   beforeEach(async () => {
     // Setup mock SQL storage
     mockSql = {
       exec: (query: string, ...bindings: any[]) => {
-        // Mock SQL implementation
-      }
+        // Mock SQL implementation for vfs_pages(file_path, page_no, page_data)
+      },
     }
-    
-    vfs = new CloudflareSqlVFS('test-vfs', mockSql, {})
+
+    vfs = new CloudflareDurableObjectVFS('test-vfs', mockSql, {})
     await vfs.isReady()
   })
 
@@ -122,6 +130,7 @@ describe('SQL VFS Test Suite', () => {
 ```
 
 ### Async Storage Test Pattern
+
 ```typescript
 import { describe, it, expect, beforeEach } from 'vitest'
 import { type Cf, CloudflareWorkerVFS } from '../../mod.ts'
@@ -133,12 +142,18 @@ describe('Async Storage VFS Test Suite', () => {
   beforeEach(async () => {
     // Setup mock DurableObjectStorage
     mockStorage = {
-      get: async (key) => { /* mock implementation */ },
-      put: async (key, value) => { /* mock implementation */ },
-      delete: async (key) => { /* mock implementation */ },
+      get: async (key) => {
+        /* mock implementation */
+      },
+      put: async (key, value) => {
+        /* mock implementation */
+      },
+      delete: async (key) => {
+        /* mock implementation */
+      },
       // ... other methods
     }
-    
+
     vfs = new CloudflareWorkerVFS('test-vfs', mockStorage, {})
     await vfs.isReady()
   })
@@ -150,12 +165,14 @@ describe('Async Storage VFS Test Suite', () => {
 ## Performance Benchmarks
 
 ### Metrics Tracked
+
 - **Throughput**: Operations per second
-- **Latency**: Response times for various operations  
+- **Latency**: Response times for various operations
 - **Memory Usage**: Cache effectiveness and memory consumption
 - **Storage Efficiency**: Data compression and chunking effectiveness
 
 ### Benchmark Comparisons
+
 - SQL vs Async Storage performance characteristics
 - Memory usage patterns between implementations
 - Scalability under different workloads
@@ -163,31 +180,35 @@ describe('Async Storage VFS Test Suite', () => {
 ## Test Data Cleanup
 
 ### Automatic Cleanup
-- **Isolated Storage**: Each test gets fresh storage instances
+
+- **Fresh mock storage**: Each test creates new in-memory storage instances
 - **Temporary Files**: Cleaned up automatically
 - **Cache Management**: Memory caches cleared between tests
 
 ## Debugging Tests
 
 ### Logging
+
 - Use `console.log` for debugging (visible in test output)
 - VFS statistics available via `vfs.getStats()`
 - Storage operations logged in development mode
 
 ### Common Issues
-1. **SQL Schema Issues**: Ensure proper table creation and constraints
+
+1. **SQL Schema Issues**: Ensure `vfs_pages(file_path, page_no, page_data)` is created as expected
 2. **Cache Misses**: Verify proper preloading in VFS initialization
-3. **Async/Sync Mismatch**: Check async operations are properly handled
+3. **Handle/Path Mismatch**: Verify reads and writes use the path captured during `jOpen`
 4. **Storage Limits**: Verify chunk sizes and storage capacity
 
 ### Test Debugging
+
 ```typescript
 // SQL VFS debugging
 console.log('Query log:', queryLog)
 const stats = vfs.getStats()
 console.log('VFS Stats:', stats)
 
-// Async Storage VFS debugging  
+// Async Storage VFS debugging
 const metadata = await storage.get('file:/test/file.db:meta')
 console.log('File metadata:', metadata)
 ```
@@ -195,12 +216,15 @@ console.log('File metadata:', metadata)
 ## Future Enhancements
 
 ### Planned Test Additions
+
 1. **Performance Comparisons**: Direct SQL vs Async Storage benchmarks
 2. **Migration Tests**: Converting between storage backends
 3. **Stress Tests**: High-load scenarios for both implementations
 4. **Real SQLite Integration**: Tests with actual SQLite WASM
+5. **Workers Runtime Coverage**: Add dedicated workerd-based tests if runtime bindings or Wrangler configuration need validation
 
 ### Implementation Improvements
+
 1. **Hybrid Approach**: Combining SQL and async storage benefits
 2. **Compression Testing**: Data compression effectiveness
 3. **Caching Strategies**: Cross-implementation cache optimization
@@ -208,6 +232,7 @@ console.log('File metadata:', metadata)
 ## Contributing
 
 When adding new tests:
+
 1. Choose the appropriate location:
    - SQL storage tests: `/src/cf/test/sql/`
    - Async storage tests: `/src/cf/test/async-storage/`
@@ -217,8 +242,9 @@ When adding new tests:
 5. Update this documentation
 
 ### Test Guidelines
+
 - **Isolation**: Each test should be independent
 - **Implementation Specific**: Tests should target specific VFS features
 - **Assertions**: Use meaningful assertions with clear error messages
 - **Coverage**: Aim for comprehensive coverage of edge cases
-- **Performance**: Consider performance implications of each approach
+- **Performance**: Consider performance implications of each approach

package/src/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.ts

@@ -1,8 +1,10 @@
 /// <reference types="vitest/globals" />
 
+import { beforeEach, describe, expect, it } from 'vitest'
+
 import type { CfTypes } from '@livestore/common-cf'
 import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
-import { beforeEach, describe, expect, it } from 'vitest'
+
 import { CloudflareWorkerVFS } from '../../mod.ts'
 
 describe('CloudflareWorkerVFS - Advanced Features', () => {
@@ -15,7 +17,7 @@ describe('CloudflareWorkerVFS - Advanced Features', () => {
 
     mockStorage = {
       get: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           return new Map()
         }
         return storageData.get(_key)
@@ -32,10 +34,10 @@ describe('CloudflareWorkerVFS - Advanced Features', () => {
       },
 
       delete: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           let count = 0
           for (const k of _key) {
-            if (storageData.delete(k)) count++
+            if (storageData.delete(k) === true) count++
           }
           return count
         } else {
@@ -156,7 +158,8 @@ describe('CloudflareWorkerVFS - Advanced Features', () => {
       expect(vfs.jClose(fileId)).toBe(VFS.SQLITE_OK)
     })
 
-    it('should handle large files that exceed cache capacity', async () => {
+    // TODO: Remove retry once flaky timeout is investigated (https://github.com/livestorejs/livestore/issues/1020)
+    it('should handle large files that exceed cache capacity', { retry: 3, timeout: 10000 }, async () => {
       const path = '/test/cache-overflow.db'
       const fileId = 1
       const flags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE

package/src/cf/test/async-storage/cloudflare-worker-vfs-core.test.ts

@@ -1,8 +1,10 @@
 /// <reference types="vitest/globals" />
 
+import { beforeEach, describe, expect, it } from 'vitest'
+
 import type { CfTypes } from '@livestore/common-cf'
 import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
-import { beforeEach, describe, expect, it } from 'vitest'
+
 import { CloudflareWorkerVFS } from '../../mod.ts'
 
 describe('CloudflareWorkerVFS - Core Functionality', () => {
@@ -15,7 +17,7 @@ describe('CloudflareWorkerVFS - Core Functionality', () => {
 
     mockStorage = {
       get: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           return new Map()
         }
         return storageData.get(_key)
@@ -32,10 +34,10 @@ describe('CloudflareWorkerVFS - Core Functionality', () => {
       },
 
       delete: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           let count = 0
           for (const k of _key) {
-            if (storageData.delete(k)) count++
+            if (storageData.delete(k) === true) count++
           }
           return count
         } else {

package/src/cf/test/async-storage/cloudflare-worker-vfs-integration.test.ts

@@ -1,8 +1,10 @@
 /// <reference types="vitest/globals" />
 
+import { beforeEach, describe, expect, it } from 'vitest'
+
 import type { CfTypes } from '@livestore/common-cf'
 import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
-import { beforeEach, describe, expect, it } from 'vitest'
+
 import { CloudflareWorkerVFS } from '../../mod.ts'
 
 describe('CloudflareWorkerVFS - Integration Tests', () => {
@@ -17,7 +19,7 @@ describe('CloudflareWorkerVFS - Integration Tests', () => {
 
     mockStorage = {
       get: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           storageOperations.push(`get-batch: ${_key.length} keys`)
           const result = new Map()
           for (const k of _key) {
@@ -46,11 +48,11 @@ describe('CloudflareWorkerVFS - Integration Tests', () => {
       },
 
       delete: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           storageOperations.push(`delete-batch: ${_key.length} keys`)
           let count = 0
           for (const k of _key) {
-            if (storageData.delete(k)) count++
+            if (storageData.delete(k) === true) count++
           }
           return count
         } else {

package/src/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.ts

@@ -1,8 +1,10 @@
 /// <reference types="vitest/globals" />
 
+import { beforeEach, describe, expect, it } from 'vitest'
+
 import type { CfTypes } from '@livestore/common-cf'
 import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
-import { beforeEach, describe, expect, it } from 'vitest'
+
 import { CloudflareWorkerVFS } from '../../mod.ts'
 
 describe('CloudflareWorkerVFS - Reliability & Error Recovery', () => {
@@ -15,7 +17,7 @@ describe('CloudflareWorkerVFS - Reliability & Error Recovery', () => {
 
     mockStorage = {
       get: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           return new Map()
         }
         return storageData.get(_key)
@@ -32,10 +34,10 @@ describe('CloudflareWorkerVFS - Reliability & Error Recovery', () => {
       },
 
       delete: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           let count = 0
           for (const k of _key) {
-            if (storageData.delete(k)) count++
+            if (storageData.delete(k) === true) count++
           }
           return count
         } else {