@livestore/sqlite-wasm 0.4.0-dev.1 → 0.4.0-dev.10

package/src/cf/README.md

@@ -0,0 +1,60 @@
+# CF SQLite WASM
+
+## VFS SQL Schema for Cloudflare Durable Object SQL Storage
+
+This schema implements a block-based file storage system on top of SQLite
+
+## Schema
+
+```sql
+
+-- File metadata table
+-- Stores information about each virtual file in the VFS
+CREATE TABLE IF NOT EXISTS vfs_files (
+  file_path TEXT PRIMARY KEY,
+  file_size INTEGER NOT NULL DEFAULT 0,
+  flags INTEGER NOT NULL DEFAULT 0,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+  modified_at INTEGER NOT NULL DEFAULT (unixepoch())
+);
+
+-- Block-based file data storage
+-- Files are split into fixed-size blocks for efficient I/O operations
+CREATE TABLE IF NOT EXISTS vfs_blocks (
+  file_path TEXT NOT NULL,
+  block_id INTEGER NOT NULL,
+  block_data BLOB NOT NULL,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+  PRIMARY KEY (file_path, block_id),
+  FOREIGN KEY (file_path) REFERENCES vfs_files(file_path) ON DELETE CASCADE
+);
+
+-- Index for efficient block range queries
+-- Essential for performance when reading/writing sequential blocks
+CREATE INDEX IF NOT EXISTS idx_vfs_blocks_range ON vfs_blocks(file_path, block_id);
+
+-- Index for file metadata queries
+CREATE INDEX IF NOT EXISTS idx_vfs_files_modified ON vfs_files(modified_at);
+
+-- Trigger to update modified_at timestamp when file size changes
+CREATE TRIGGER IF NOT EXISTS trg_vfs_files_update_modified 
+  AFTER UPDATE OF file_size ON vfs_files
+  BEGIN
+    UPDATE vfs_files SET modified_at = unixepoch() WHERE file_path = NEW.file_path;
+  END;
+
+-- View for file statistics (useful for debugging and monitoring)
+CREATE VIEW IF NOT EXISTS vfs_file_stats AS
+SELECT 
+  f.file_path,
+  f.file_size,
+  f.flags,
+  f.created_at,
+  f.modified_at,
+  COUNT(b.block_id) as block_count,
+  COALESCE(SUM(LENGTH(b.block_data)), 0) as stored_bytes,
+  ROUND(CAST(COALESCE(SUM(LENGTH(b.block_data)), 0) AS REAL) / NULLIF(f.file_size, 0) * 100, 2) as compression_ratio
+FROM vfs_files f
+LEFT JOIN vfs_blocks b ON f.file_path = b.file_path
+GROUP BY f.file_path, f.file_size, f.flags, f.created_at, f.modified_at;
+```

package/src/cf/mod.ts

@@ -0,0 +1,143 @@
+// import path from 'node:path'
+
+import { type MakeSqliteDb, type PersistenceInfo, type SqliteDb, UnexpectedError } from '@livestore/common'
+import type { CfTypes } from '@livestore/common-cf'
+import { Effect } from '@livestore/utils/effect'
+import type * as WaSqlite from '@livestore/wa-sqlite'
+import type { MemoryVFS } from '@livestore/wa-sqlite/src/examples/MemoryVFS.js'
+import { makeInMemoryDb } from '../in-memory-vfs.ts'
+import { makeSqliteDb } from '../make-sqlite-db.ts'
+import { CloudflareSqlVFS } from './CloudflareSqlVFS.ts'
+
+export { BlockManager } from './BlockManager.ts'
+export { CloudflareSqlVFS } from './CloudflareSqlVFS.ts'
+export { CloudflareWorkerVFS } from './CloudflareWorkerVFS.ts'
+
+export type CloudflareDatabaseMetadataInMemory = {
+  _tag: 'in-memory'
+  vfs: MemoryVFS
+  dbPointer: number
+  persistenceInfo: PersistenceInfo
+  deleteDb: () => void
+  configureDb: (db: SqliteDb) => void
+}
+
+export type CloudflareDatabaseMetadataFs = {
+  _tag: 'storage'
+  // vfs: CloudflareWorkerVFS
+  vfs: CloudflareSqlVFS
+  dbPointer: number
+  persistenceInfo: PersistenceInfo
+  deleteDb: () => void
+  configureDb: (db: SqliteDb) => void
+}
+
+export type CloudflareDatabaseMetadata = CloudflareDatabaseMetadataInMemory | CloudflareDatabaseMetadataFs
+
+export type CloudflareDatabaseInputInMemory = {
+  _tag: 'in-memory'
+  configureDb?: (db: SqliteDb) => void
+}
+
+export type CloudflareDatabaseInputFs = {
+  _tag: 'storage'
+  // directory: string
+  fileName: string
+  configureDb?: (db: SqliteDb) => void
+  storage: CfTypes.DurableObjectStorage
+}
+
+export type CloudflareDatabaseInput = CloudflareDatabaseInputInMemory | CloudflareDatabaseInputFs
+
+export type MakeCloudflareSqliteDb = MakeSqliteDb<
+  { dbPointer: number; persistenceInfo: PersistenceInfo },
+  CloudflareDatabaseInput,
+  CloudflareDatabaseMetadata
+>
+
+export const sqliteDbFactory =
+  ({ sqlite3 }: { sqlite3: SQLiteAPI }): MakeCloudflareSqliteDb =>
+  (input) =>
+    Effect.gen(function* () {
+      if (input._tag === 'in-memory') {
+        const { dbPointer, vfs } = makeInMemoryDb(sqlite3)
+        return makeSqliteDb<CloudflareDatabaseMetadataInMemory>({
+          sqlite3,
+          metadata: {
+            _tag: 'in-memory',
+            vfs,
+            dbPointer,
+            persistenceInfo: { fileName: ':memory:' },
+            deleteDb: () => {},
+            configureDb: input.configureDb ?? (() => {}),
+          },
+        }) as any
+      }
+
+      const { dbPointer, vfs } = yield* makeCloudflareFsDb({
+        sqlite3,
+        fileName: input.fileName,
+        // directory: input.directory,
+        storage: input.storage,
+      })
+
+      // const filePath = path.join(input.directory, input.fileName)
+      // const filePath = `${input.directory}/${input.fileName}`
+
+      return makeSqliteDb<CloudflareDatabaseMetadataFs>({
+        sqlite3,
+        metadata: {
+          _tag: 'storage',
+          vfs,
+          dbPointer,
+          persistenceInfo: { fileName: input.fileName },
+          // deleteDb: () => vfs.deleteDb(filePath),
+          // TODO: implement deleteDb
+          deleteDb: () => {},
+          configureDb: input.configureDb ?? (() => {}),
+        },
+      })
+    })
+
+const makeCloudflareFsDb = ({
+  sqlite3,
+  fileName,
+  // directory,
+  storage,
+}: {
+  sqlite3: WaSqlite.SQLiteAPI
+  fileName: string
+  // directory: string
+  storage: CfTypes.DurableObjectStorage
+}) =>
+  Effect.gen(function* () {
+    // NOTE to keep the filePath short, we use the directory name in the vfs name
+    // If this is becoming a problem, we can use a hashed version of the directory name
+    const vfsName = `cf-do-sqlite-${fileName}`
+    if (sqlite3.vfs_registered.has(vfsName) === false) {
+      // TODO refactor with Effect FileSystem instead of using `node:fs` directly inside of CloudflareWorkerVFS
+      // const nodeFsVfs = new CloudflareWorkerVFS(vfsName, storage, (sqlite3 as any).module)
+      const nodeFsVfs = new CloudflareSqlVFS(vfsName, storage.sql, (sqlite3 as any).module)
+
+      // Initialize the VFS schema before registering it
+      const isReady = yield* Effect.promise(() => nodeFsVfs.isReady())
+      if (!isReady) {
+        throw new Error(`Failed to initialize CloudflareSqlVFS for ${vfsName}`)
+      }
+
+      // @ts-expect-error TODO fix types
+      sqlite3.vfs_register(nodeFsVfs, false)
+    }
+
+    // yield* fs.makeDirectory(directory, { recursive: true })
+
+    const FILE_NAME_MAX_LENGTH = 56
+    if (fileName.length > FILE_NAME_MAX_LENGTH) {
+      throw new Error(`File name ${fileName} is too long. Maximum length is ${FILE_NAME_MAX_LENGTH} characters.`)
+    }
+
+    // NOTE SQLite will return a "disk I/O error" if the file path is too long.
+    const dbPointer = sqlite3.open_v2Sync(fileName, undefined, vfsName)
+
+    return { dbPointer, vfs: {} as UNUSED<'only needed in web adapter currently and should longer-term be removed'> }
+  }).pipe(UnexpectedError.mapToUnexpectedError)

package/src/cf/test/README.md

@@ -0,0 +1,224 @@
+# Cloudflare SQLite WASM Test Suite
+
+This directory contains comprehensive tests for both SQLite WASM VFS implementations on Cloudflare Workers.
+
+## Directory Structure
+
+### `/sql/` - SQL Storage Tests
+Tests for the CloudflareSqlVFS implementation, which uses Cloudflare's DurableObject SQL API for storage.
+
+- **`cloudflare-sql-vfs-core.test.ts`** - Core SQL VFS functionality tests
+
+### `/async-storage/` - Async Storage Tests  
+Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorage (key-value) for file storage.
+
+- **`cloudflare-worker-vfs-core.test.ts`** - Basic VFS operations (open, read, write, close, sync)
+- **`cloudflare-worker-vfs-advanced.test.ts`** - Large file chunking and advanced features
+- **`cloudflare-worker-vfs-reliability.test.ts`** - Error recovery and reliability testing
+- **`cloudflare-worker-vfs-integration.test.ts`** - End-to-end integration tests
+
+## Test Architecture
+
+### Testing Framework
+- **Vitest** with **@cloudflare/vitest-pool-workers** for Workers runtime testing
+- **Isolated Storage** - Each test gets fresh storage instances
+- **Real Runtime** - Tests run in the actual Cloudflare Workers runtime environment
+
+## VFS Implementation Comparison
+
+### SQL Storage VFS (`CloudflareSqlVFS`)
+- **Backend**: Cloudflare DurableObject SQL API
+- **Storage Model**: Relational tables with blocks and metadata
+- **Advantages**: ACID transactions, complex queries, relational integrity
+- **Use Case**: Applications requiring complex data relationships
+
+### Async Storage VFS (`CloudflareWorkerVFS`)  
+- **Backend**: DurableObjectStorage (key-value)
+- **Storage Model**: 64 KiB chunks with LRU caching
+- **Advantages**: Simple API, optimized for large files, better caching
+- **Use Case**: High-performance file operations, large databases
+
+## Key Design Decisions Tested
+
+### SQL Storage Approach
+- **Block-based Storage**: Files stored as blocks in SQL tables
+- **Metadata Management**: File metadata in dedicated tables
+- **Transaction Safety**: ACID compliance for data integrity
+
+### Async Storage Approach
+- **64 KiB Chunking Strategy**: Optimized for SQLite I/O patterns
+- **Synchronous Interface with Async Backend**: Aggressive caching + background sync
+- **Memory Management**: LRU cache for chunks, complete metadata cache
+- **Error Handling**: SQLite-compatible error codes
+
+## Running Tests
+
+### Prerequisites
+1. Install dependencies:
+   ```bash
+   pnpm install
+   ```
+
+2. Ensure Wrangler configuration is correct:
+   ```bash
+   # Check wrangler.toml exists and has correct Durable Object bindings
+   ```
+
+### Test Commands
+
+```bash
+# Run all tests
+pnpm test
+
+# Run tests in watch mode
+pnpm test:watch
+
+# Run tests with UI
+pnpm test:ui
+
+# Run specific test directory
+pnpm test src/cf/                         # All cf tests (SQL + async storage)
+pnpm test src/cf/test/sql/                # SQL storage tests only
+pnpm test src/cf/test/async-storage/      # Async storage tests only
+
+# Run specific test file
+pnpm test cloudflare-sql-vfs-core.test.ts
+
+# Run with coverage
+pnpm test --coverage
+```
+
+### Test Environment
+
+- **Runtime**: Cloudflare Workers (via workerd)
+- **Storage**: Isolated per-test storage instances
+
+## Test Structure
+
+### SQL Storage Test Pattern
+```typescript
+import { describe, it, expect, beforeEach } from 'vitest'
+import { type Cf, CloudflareSqlVFS } from '../../mod.ts'
+
+describe('SQL VFS Test Suite', () => {
+  let vfs: CloudflareSqlVFS
+  let mockSql: Cf.SqlStorage
+  let queryLog: string[]
+
+  beforeEach(async () => {
+    // Setup mock SQL storage
+    mockSql = {
+      exec: (query: string, ...bindings: any[]) => {
+        // Mock SQL implementation
+      }
+    }
+    
+    vfs = new CloudflareSqlVFS('test-vfs', mockSql, {})
+    await vfs.isReady()
+  })
+
+  // Tests...
+})
+```
+
+### Async Storage Test Pattern
+```typescript
+import { describe, it, expect, beforeEach } from 'vitest'
+import { type Cf, CloudflareWorkerVFS } from '../../mod.ts'
+
+describe('Async Storage VFS Test Suite', () => {
+  let vfs: CloudflareWorkerVFS
+  let mockStorage: DurableObjectStorage
+
+  beforeEach(async () => {
+    // Setup mock DurableObjectStorage
+    mockStorage = {
+      get: async (key) => { /* mock implementation */ },
+      put: async (key, value) => { /* mock implementation */ },
+      delete: async (key) => { /* mock implementation */ },
+      // ... other methods
+    }
+    
+    vfs = new CloudflareWorkerVFS('test-vfs', mockStorage, {})
+    await vfs.isReady()
+  })
+
+  // Tests...
+})
+```
+
+## Performance Benchmarks
+
+### Metrics Tracked
+- **Throughput**: Operations per second
+- **Latency**: Response times for various operations  
+- **Memory Usage**: Cache effectiveness and memory consumption
+- **Storage Efficiency**: Data compression and chunking effectiveness
+
+### Benchmark Comparisons
+- SQL vs Async Storage performance characteristics
+- Memory usage patterns between implementations
+- Scalability under different workloads
+
+## Test Data Cleanup
+
+### Automatic Cleanup
+- **Isolated Storage**: Each test gets fresh storage instances
+- **Temporary Files**: Cleaned up automatically
+- **Cache Management**: Memory caches cleared between tests
+
+## Debugging Tests
+
+### Logging
+- Use `console.log` for debugging (visible in test output)
+- VFS statistics available via `vfs.getStats()`
+- Storage operations logged in development mode
+
+### Common Issues
+1. **SQL Schema Issues**: Ensure proper table creation and constraints
+2. **Cache Misses**: Verify proper preloading in VFS initialization
+3. **Async/Sync Mismatch**: Check async operations are properly handled
+4. **Storage Limits**: Verify chunk sizes and storage capacity
+
+### Test Debugging
+```typescript
+// SQL VFS debugging
+console.log('Query log:', queryLog)
+const stats = vfs.getStats()
+console.log('VFS Stats:', stats)
+
+// Async Storage VFS debugging  
+const metadata = await storage.get('file:/test/file.db:meta')
+console.log('File metadata:', metadata)
+```
+
+## Future Enhancements
+
+### Planned Test Additions
+1. **Performance Comparisons**: Direct SQL vs Async Storage benchmarks
+2. **Migration Tests**: Converting between storage backends
+3. **Stress Tests**: High-load scenarios for both implementations
+4. **Real SQLite Integration**: Tests with actual SQLite WASM
+
+### Implementation Improvements
+1. **Hybrid Approach**: Combining SQL and async storage benefits
+2. **Compression Testing**: Data compression effectiveness
+3. **Caching Strategies**: Cross-implementation cache optimization
+
+## Contributing
+
+When adding new tests:
+1. Choose the appropriate location:
+   - SQL storage tests: `/src/cf/test/sql/`
+   - Async storage tests: `/src/cf/test/async-storage/`
+2. Follow the existing test structure for that implementation
+3. Use descriptive test names indicating the VFS type
+4. Include both positive and negative test cases
+5. Update this documentation
+
+### Test Guidelines
+- **Isolation**: Each test should be independent
+- **Implementation Specific**: Tests should target specific VFS features
+- **Assertions**: Use meaningful assertions with clear error messages
+- **Coverage**: Aim for comprehensive coverage of edge cases
+- **Performance**: Consider performance implications of each approach