@livestore/sqlite-wasm 0.4.0-dev.22 → 0.4.0-dev.24

package/src/cf/test/README.md

@@ -5,11 +5,13 @@ This directory contains comprehensive tests for both SQLite WASM VFS implementat
 ## Directory Structure
 
 ### `/sql/` - SQL Storage Tests
-Tests for the CloudflareSqlVFS implementation, which uses Cloudflare's DurableObject SQL API for storage.
+
+Tests for the CloudflareDurableObjectVFS implementation, which uses Cloudflare's [SQLite in DurableObjects](https://developers.cloudflare.com/durable-objects/api/sql-storage/) for storage.
 
 - **`cloudflare-sql-vfs-core.test.ts`** - Core SQL VFS functionality tests
 
-### `/async-storage/` - Async Storage Tests  
+### `/async-storage/` - Async Storage Tests
+
 Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorage (key-value) for file storage.
 
 - **`cloudflare-worker-vfs-core.test.ts`** - Basic VFS operations (open, read, write, close, sync)
@@ -20,19 +22,22 @@ Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorag
 ## Test Architecture
 
 ### Testing Framework
+
 - **Vitest** with **@cloudflare/vitest-pool-workers** for Workers runtime testing
 - **Isolated Storage** - Each test gets fresh storage instances
 - **Real Runtime** - Tests run in the actual Cloudflare Workers runtime environment
 
 ## VFS Implementation Comparison
 
-### SQL Storage VFS (`CloudflareSqlVFS`)
-- **Backend**: Cloudflare DurableObject SQL API
-- **Storage Model**: Relational tables with blocks and metadata
-- **Advantages**: ACID transactions, complex queries, relational integrity
+### SQL Storage VFS (`CloudflareDurableObjectVFS`)
+
+- **Backend**: Cloudflare's SQLite in Durable Objects
+- **Storage Model**: Single `vfs_pages` table keyed by file path and page number
+- **Advantages**: Simple synchronous SQL storage with direct page lookups
 - **Use Case**: Applications requiring complex data relationships
 
-### Async Storage VFS (`CloudflareWorkerVFS`)  
+### Async Storage VFS (`CloudflareWorkerVFS`)
+
 - **Backend**: DurableObjectStorage (key-value)
 - **Storage Model**: 64 KiB chunks with LRU caching
 - **Advantages**: Simple API, optimized for large files, better caching
@@ -41,11 +46,14 @@ Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorag
 ## Key Design Decisions Tested
 
 ### SQL Storage Approach
-- **Block-based Storage**: Files stored as blocks in SQL tables
-- **Metadata Management**: File metadata in dedicated tables
-- **Transaction Safety**: ACID compliance for data integrity
+
+- **Page-based Storage**: SQLite pages stored directly in `vfs_pages`
+- **File-path Isolation**: Each logical SQLite file is scoped by `file_path`
+- **Handle Tracking**: Open SQLite file handles map back to persisted file paths
+- **Transaction Model**: Page writes are persisted directly through Durable Object SQL
 
 ### Async Storage Approach
+
 - **64 KiB Chunking Strategy**: Optimized for SQLite I/O patterns
 - **Synchronous Interface with Async Backend**: Aggressive caching + background sync
 - **Memory Management**: LRU cache for chunks, complete metadata cache
@@ -54,7 +62,9 @@ Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorag
 ## Running Tests
 
 ### Prerequisites
+
 1. Install dependencies:
+
    ```bash
    pnpm install
    ```
@@ -96,24 +106,26 @@ pnpm test --coverage
 ## Test Structure
 
 ### SQL Storage Test Pattern
+
 ```typescript
 import { describe, it, expect, beforeEach } from 'vitest'
-import { type Cf, CloudflareSqlVFS } from '../../mod.ts'
+import type { CfTypes } from '@livestore/common-cf'
+import { CloudflareDurableObjectVFS } from '../../mod.ts'
 
 describe('SQL VFS Test Suite', () => {
-  let vfs: CloudflareSqlVFS
-  let mockSql: Cf.SqlStorage
+  let vfs: CloudflareDurableObjectVFS
+  let mockSql: CfTypes.SqlStorage
   let queryLog: string[]
 
   beforeEach(async () => {
     // Setup mock SQL storage
     mockSql = {
       exec: (query: string, ...bindings: any[]) => {
-        // Mock SQL implementation
-      }
+        // Mock SQL implementation for vfs_pages(file_path, page_no, page_data)
+      },
     }
-    
-    vfs = new CloudflareSqlVFS('test-vfs', mockSql, {})
+
+    vfs = new CloudflareDurableObjectVFS('test-vfs', mockSql, {})
     await vfs.isReady()
   })
 
@@ -122,6 +134,7 @@ describe('SQL VFS Test Suite', () => {
 ```
 
 ### Async Storage Test Pattern
+
 ```typescript
 import { describe, it, expect, beforeEach } from 'vitest'
 import { type Cf, CloudflareWorkerVFS } from '../../mod.ts'
@@ -133,12 +146,18 @@ describe('Async Storage VFS Test Suite', () => {
   beforeEach(async () => {
     // Setup mock DurableObjectStorage
     mockStorage = {
-      get: async (key) => { /* mock implementation */ },
-      put: async (key, value) => { /* mock implementation */ },
-      delete: async (key) => { /* mock implementation */ },
+      get: async (key) => {
+        /* mock implementation */
+      },
+      put: async (key, value) => {
+        /* mock implementation */
+      },
+      delete: async (key) => {
+        /* mock implementation */
+      },
       // ... other methods
     }
-    
+
     vfs = new CloudflareWorkerVFS('test-vfs', mockStorage, {})
     await vfs.isReady()
   })
@@ -150,12 +169,14 @@ describe('Async Storage VFS Test Suite', () => {
 ## Performance Benchmarks
 
 ### Metrics Tracked
+
 - **Throughput**: Operations per second
-- **Latency**: Response times for various operations  
+- **Latency**: Response times for various operations
 - **Memory Usage**: Cache effectiveness and memory consumption
 - **Storage Efficiency**: Data compression and chunking effectiveness
 
 ### Benchmark Comparisons
+
 - SQL vs Async Storage performance characteristics
 - Memory usage patterns between implementations
 - Scalability under different workloads
@@ -163,6 +184,7 @@ describe('Async Storage VFS Test Suite', () => {
 ## Test Data Cleanup
 
 ### Automatic Cleanup
+
 - **Isolated Storage**: Each test gets fresh storage instances
 - **Temporary Files**: Cleaned up automatically
 - **Cache Management**: Memory caches cleared between tests
@@ -170,24 +192,27 @@ describe('Async Storage VFS Test Suite', () => {
 ## Debugging Tests
 
 ### Logging
+
 - Use `console.log` for debugging (visible in test output)
 - VFS statistics available via `vfs.getStats()`
 - Storage operations logged in development mode
 
 ### Common Issues
-1. **SQL Schema Issues**: Ensure proper table creation and constraints
+
+1. **SQL Schema Issues**: Ensure `vfs_pages(file_path, page_no, page_data)` is created as expected
 2. **Cache Misses**: Verify proper preloading in VFS initialization
-3. **Async/Sync Mismatch**: Check async operations are properly handled
+3. **Handle/Path Mismatch**: Verify reads and writes use the path captured during `jOpen`
 4. **Storage Limits**: Verify chunk sizes and storage capacity
 
 ### Test Debugging
+
 ```typescript
 // SQL VFS debugging
 console.log('Query log:', queryLog)
 const stats = vfs.getStats()
 console.log('VFS Stats:', stats)
 
-// Async Storage VFS debugging  
+// Async Storage VFS debugging
 const metadata = await storage.get('file:/test/file.db:meta')
 console.log('File metadata:', metadata)
 ```
@@ -195,12 +220,14 @@ console.log('File metadata:', metadata)
 ## Future Enhancements
 
 ### Planned Test Additions
+
 1. **Performance Comparisons**: Direct SQL vs Async Storage benchmarks
 2. **Migration Tests**: Converting between storage backends
 3. **Stress Tests**: High-load scenarios for both implementations
 4. **Real SQLite Integration**: Tests with actual SQLite WASM
 
 ### Implementation Improvements
+
 1. **Hybrid Approach**: Combining SQL and async storage benefits
 2. **Compression Testing**: Data compression effectiveness
 3. **Caching Strategies**: Cross-implementation cache optimization
@@ -208,6 +235,7 @@ console.log('File metadata:', metadata)
 ## Contributing
 
 When adding new tests:
+
 1. Choose the appropriate location:
    - SQL storage tests: `/src/cf/test/sql/`
    - Async storage tests: `/src/cf/test/async-storage/`
@@ -217,8 +245,9 @@ When adding new tests:
 5. Update this documentation
 
 ### Test Guidelines
+
 - **Isolation**: Each test should be independent
 - **Implementation Specific**: Tests should target specific VFS features
 - **Assertions**: Use meaningful assertions with clear error messages
 - **Coverage**: Aim for comprehensive coverage of edge cases
-- **Performance**: Consider performance implications of each approach
+- **Performance**: Consider performance implications of each approach

package/src/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.ts

@@ -3,6 +3,7 @@
 import type { CfTypes } from '@livestore/common-cf'
 import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
 import { beforeEach, describe, expect, it } from 'vitest'
+
 import { CloudflareWorkerVFS } from '../../mod.ts'
 
 describe('CloudflareWorkerVFS - Advanced Features', () => {
@@ -15,7 +16,7 @@ describe('CloudflareWorkerVFS - Advanced Features', () => {
 
     mockStorage = {
       get: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           return new Map()
         }
         return storageData.get(_key)
@@ -32,10 +33,10 @@ describe('CloudflareWorkerVFS - Advanced Features', () => {
       },
 
       delete: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           let count = 0
           for (const k of _key) {
-            if (storageData.delete(k)) count++
+            if (storageData.delete(k) === true) count++
           }
           return count
         } else {
@@ -156,7 +157,8 @@ describe('CloudflareWorkerVFS - Advanced Features', () => {
       expect(vfs.jClose(fileId)).toBe(VFS.SQLITE_OK)
     })
 
-    it('should handle large files that exceed cache capacity', async () => {
+    // TODO: Remove retry once flaky timeout is investigated (https://github.com/livestorejs/livestore/issues/1020)
+    it('should handle large files that exceed cache capacity', { retry: 3, timeout: 10000 }, async () => {
       const path = '/test/cache-overflow.db'
       const fileId = 1
       const flags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE

package/src/cf/test/async-storage/cloudflare-worker-vfs-core.test.ts

@@ -3,6 +3,7 @@
 import type { CfTypes } from '@livestore/common-cf'
 import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
 import { beforeEach, describe, expect, it } from 'vitest'
+
 import { CloudflareWorkerVFS } from '../../mod.ts'
 
 describe('CloudflareWorkerVFS - Core Functionality', () => {
@@ -15,7 +16,7 @@ describe('CloudflareWorkerVFS - Core Functionality', () => {
 
     mockStorage = {
       get: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           return new Map()
         }
         return storageData.get(_key)
@@ -32,10 +33,10 @@ describe('CloudflareWorkerVFS - Core Functionality', () => {
       },
 
       delete: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           let count = 0
           for (const k of _key) {
-            if (storageData.delete(k)) count++
+            if (storageData.delete(k) === true) count++
           }
           return count
         } else {

package/src/cf/test/async-storage/cloudflare-worker-vfs-integration.test.ts

@@ -3,6 +3,7 @@
 import type { CfTypes } from '@livestore/common-cf'
 import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
 import { beforeEach, describe, expect, it } from 'vitest'
+
 import { CloudflareWorkerVFS } from '../../mod.ts'
 
 describe('CloudflareWorkerVFS - Integration Tests', () => {
@@ -17,7 +18,7 @@ describe('CloudflareWorkerVFS - Integration Tests', () => {
 
     mockStorage = {
       get: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           storageOperations.push(`get-batch: ${_key.length} keys`)
           const result = new Map()
           for (const k of _key) {
@@ -46,11 +47,11 @@ describe('CloudflareWorkerVFS - Integration Tests', () => {
       },
 
       delete: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           storageOperations.push(`delete-batch: ${_key.length} keys`)
           let count = 0
           for (const k of _key) {
-            if (storageData.delete(k)) count++
+            if (storageData.delete(k) === true) count++
           }
           return count
         } else {

package/src/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.ts

@@ -3,6 +3,7 @@
 import type { CfTypes } from '@livestore/common-cf'
 import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
 import { beforeEach, describe, expect, it } from 'vitest'
+
 import { CloudflareWorkerVFS } from '../../mod.ts'
 
 describe('CloudflareWorkerVFS - Reliability & Error Recovery', () => {
@@ -15,7 +16,7 @@ describe('CloudflareWorkerVFS - Reliability & Error Recovery', () => {
 
     mockStorage = {
       get: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           return new Map()
         }
         return storageData.get(_key)
@@ -32,10 +33,10 @@ describe('CloudflareWorkerVFS - Reliability & Error Recovery', () => {
       },
 
       delete: (async (_key: string | string[]) => {
-        if (Array.isArray(_key)) {
+        if (Array.isArray(_key) === true) {
           let count = 0
           for (const k of _key) {
-            if (storageData.delete(k)) count++
+            if (storageData.delete(k) === true) count++
           }
           return count
         } else {