@dotdo/postgres 0.1.0

package/src/pglite/tiered-vfs.test.ts

@@ -0,0 +1,726 @@
+/**
+ * RED PHASE Tests for TieredVFS - Combine cache, DO, R2 layers
+ * Task: postgres-xm8.9 - TieredVFS - combine cache, DO, R2 layers
+ *
+ * TieredVFS provides a unified virtual file system interface that:
+ * - Reads from Cache (hot) -> DO Storage (warm) -> R2 (cold)
+ * - Writes to DO Storage, promotes to Cache on access
+ * - Demotes cold pages from Cache based on TTL/LRU
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import {
+  TieredVFS,
+  TieredVFSConfig,
+  StorageTier,
+  PageLocation,
+  TierHealth,
+  TierFailure,
+  TierHealthStatus,
+  createTieredVFS,
+} from '../storage/tiered-vfs'
+import { CacheLayer } from './cache-layer'
+import { R2StorageLayer } from './r2-layer'
+
+// Mock CacheLayer
+const mockCacheLayer = {
+  get: vi.fn(),
+  put: vi.fn(),
+  delete: vi.fn(),
+  has: vi.fn(),
+  getWithMetadata: vi.fn(),
+  getStats: vi.fn().mockReturnValue({
+    hits: 0,
+    misses: 0,
+    writes: 0,
+    deletes: 0,
+    bytesRead: 0,
+    bytesWritten: 0,
+    errors: 0,
+    hitRatio: 0,
+  }),
+  clear: vi.fn(),
+}
+
+// Mock DOStorage (Durable Object storage simulation)
+const mockDOStorage = {
+  get: vi.fn(),
+  put: vi.fn().mockResolvedValue(undefined),
+  delete: vi.fn().mockResolvedValue(undefined),
+  list: vi.fn(),
+}
+
+// Mock R2StorageLayer
+const mockR2Layer = {
+  get: vi.fn(),
+  getRange: vi.fn(),
+  put: vi.fn(),
+  delete: vi.fn(),
+  deleteMany: vi.fn(),
+  head: vi.fn(),
+  has: vi.fn(),
+  list: vi.fn(),
+  getStats: vi.fn().mockReturnValue({
+    reads: 0,
+    rangeReads: 0,
+    writes: 0,
+    deletes: 0,
+    bytesRead: 0,
+    bytesWritten: 0,
+    errors: 0,
+  }),
+}
+
+describe('TieredVFS - combine cache, DO, R2 layers', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  describe('TieredVFSConfig', () => {
+    it('should have required config fields', () => {
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      expect(config.cacheLayer).toBeDefined()
+      expect(config.doStorage).toBeDefined()
+      expect(config.r2Layer).toBeDefined()
+      expect(config.pageSize).toBe(8192)
+    })
+
+    it('should support optional promotion threshold', () => {
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+        promotionThreshold: 3, // promote after 3 accesses
+      }
+      expect(config.promotionThreshold).toBe(3)
+    })
+  })
+
+  describe('createTieredVFS()', () => {
+    it('should return a TieredVFS instance', () => {
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+      expect(vfs).toBeDefined()
+      expect(vfs).toBeInstanceOf(TieredVFS)
+    })
+  })
+
+  describe('TieredVFS.readPage() - tiered lookup', () => {
+    it('should try cache first (hot tier)', async () => {
+      const data = new Uint8Array([1, 2, 3, 4, 5])
+      mockCacheLayer.get.mockResolvedValue(data)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const result = await vfs.readPage('page-123')
+      expect(result).toEqual(data)
+      expect(mockCacheLayer.get).toHaveBeenCalledWith('page-123')
+      expect(mockDOStorage.get).not.toHaveBeenCalled()
+      expect(mockR2Layer.get).not.toHaveBeenCalled()
+    })
+
+    it('should try DO storage on cache miss (warm tier)', async () => {
+      const data = new Uint8Array([10, 20, 30])
+      mockCacheLayer.get.mockResolvedValue(null)
+      mockDOStorage.get.mockResolvedValue(data)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const result = await vfs.readPage('page-123')
+      expect(result).toEqual(data)
+      expect(mockCacheLayer.get).toHaveBeenCalled()
+      expect(mockDOStorage.get).toHaveBeenCalledWith('page-123')
+      expect(mockR2Layer.get).not.toHaveBeenCalled()
+    })
+
+    it('should try R2 on DO miss (cold tier)', async () => {
+      const data = new Uint8Array([100, 200])
+      mockCacheLayer.get.mockResolvedValue(null)
+      mockDOStorage.get.mockResolvedValue(undefined)
+      mockR2Layer.get.mockResolvedValue(data)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const result = await vfs.readPage('page-123')
+      expect(result).toEqual(data)
+      expect(mockCacheLayer.get).toHaveBeenCalled()
+      expect(mockDOStorage.get).toHaveBeenCalled()
+      expect(mockR2Layer.get).toHaveBeenCalledWith('page-123')
+    })
+
+    it('should return null when page not found in any tier', async () => {
+      mockCacheLayer.get.mockResolvedValue(null)
+      mockDOStorage.get.mockResolvedValue(undefined)
+      mockR2Layer.get.mockResolvedValue(null)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const result = await vfs.readPage('nonexistent')
+      expect(result).toBeNull()
+    })
+  })
+
+  describe('TieredVFS.writePage()', () => {
+    it('should write to DO storage by default', async () => {
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const data = new Uint8Array([1, 2, 3])
+      await vfs.writePage('page-456', data)
+
+      expect(mockDOStorage.put).toHaveBeenCalledWith('page-456', data)
+    })
+
+    it('should also write to cache for hot pages', async () => {
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const data = new Uint8Array([1, 2, 3])
+      await vfs.writePage('page-456', data, { tier: 'hot' })
+
+      expect(mockDOStorage.put).toHaveBeenCalledWith('page-456', data)
+      expect(mockCacheLayer.put).toHaveBeenCalledWith('page-456', data, expect.any(Object))
+    })
+
+    it('should write to R2 for cold tier', async () => {
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const data = new Uint8Array([1, 2, 3])
+      await vfs.writePage('page-456', data, { tier: 'cold' })
+
+      expect(mockR2Layer.put).toHaveBeenCalledWith('page-456', data, expect.any(Object))
+    })
+  })
+
+  describe('TieredVFS.deletePage()', () => {
+    it('should delete from all tiers', async () => {
+      mockCacheLayer.delete.mockResolvedValue(true)
+      mockR2Layer.delete.mockResolvedValue(undefined)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      await vfs.deletePage('page-789')
+
+      expect(mockCacheLayer.delete).toHaveBeenCalledWith('page-789')
+      expect(mockDOStorage.delete).toHaveBeenCalledWith('page-789')
+      expect(mockR2Layer.delete).toHaveBeenCalledWith('page-789')
+    })
+  })
+
+  describe('TieredVFS.locatePage()', () => {
+    it('should return hot tier when in cache', async () => {
+      mockCacheLayer.has.mockResolvedValue(true)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const location = await vfs.locatePage('page-123')
+      expect(location.tier).toBe('hot')
+      expect(location.found).toBe(true)
+    })
+
+    it('should return warm tier when in DO', async () => {
+      mockCacheLayer.has.mockResolvedValue(false)
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const location = await vfs.locatePage('page-123')
+      expect(location.tier).toBe('warm')
+      expect(location.found).toBe(true)
+    })
+
+    it('should return cold tier when in R2', async () => {
+      mockCacheLayer.has.mockResolvedValue(false)
+      mockDOStorage.get.mockResolvedValue(undefined)
+      mockR2Layer.has.mockResolvedValue(true)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const location = await vfs.locatePage('page-123')
+      expect(location.tier).toBe('cold')
+      expect(location.found).toBe(true)
+    })
+
+    it('should return not found when page does not exist', async () => {
+      mockCacheLayer.has.mockResolvedValue(false)
+      mockDOStorage.get.mockResolvedValue(undefined)
+      mockR2Layer.has.mockResolvedValue(false)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const location = await vfs.locatePage('nonexistent')
+      expect(location.found).toBe(false)
+    })
+  })
+
+  describe('TieredVFS.getStats()', () => {
+    it('should return combined stats from all layers', () => {
+      mockCacheLayer.getStats.mockReturnValue({
+        hits: 100,
+        misses: 50,
+        writes: 30,
+        deletes: 5,
+        bytesRead: 10000,
+        bytesWritten: 5000,
+        errors: 1,
+        hitRatio: 0.67,
+      })
+      mockR2Layer.getStats.mockReturnValue({
+        reads: 20,
+        rangeReads: 5,
+        writes: 10,
+        deletes: 2,
+        bytesRead: 2000,
+        bytesWritten: 1000,
+        errors: 0,
+      })
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const stats = vfs.getStats()
+      expect(stats.cache).toBeDefined()
+      expect(stats.r2).toBeDefined()
+      expect(stats.cache.hits).toBe(100)
+      expect(stats.r2.reads).toBe(20)
+    })
+  })
+
+  describe('StorageTier type', () => {
+    it('should define three tiers', () => {
+      const hot: StorageTier = 'hot'
+      const warm: StorageTier = 'warm'
+      const cold: StorageTier = 'cold'
+      expect(hot).toBe('hot')
+      expect(warm).toBe('warm')
+      expect(cold).toBe('cold')
+    })
+  })
+
+  describe('PageLocation type', () => {
+    it('should have tier and found properties', () => {
+      const location: PageLocation = {
+        tier: 'hot',
+        found: true,
+        key: 'page-123',
+      }
+      expect(location.tier).toBe('hot')
+      expect(location.found).toBe(true)
+      expect(location.key).toBe('page-123')
+    })
+  })
+
+  // Graceful degradation tests (postgres-avbp)
+  describe('Graceful degradation - readPage() with tier failures', () => {
+    it('should fall back to DO when cache fails', async () => {
+      const data = new Uint8Array([10, 20, 30])
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockResolvedValue(data)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const result = await vfs.readPage('page-123')
+      expect(result).toEqual(data)
+      expect(mockCacheLayer.get).toHaveBeenCalled()
+      expect(mockDOStorage.get).toHaveBeenCalledWith('page-123')
+    })
+
+    it('should fall back to R2 when cache and DO fail', async () => {
+      const data = new Uint8Array([100, 200])
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockRejectedValue(new Error('DO unavailable'))
+      mockR2Layer.get.mockResolvedValue(data)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const result = await vfs.readPage('page-123')
+      expect(result).toEqual(data)
+      expect(mockR2Layer.get).toHaveBeenCalledWith('page-123')
+    })
+
+    it('should return null when all tiers fail', async () => {
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockRejectedValue(new Error('DO unavailable'))
+      mockR2Layer.get.mockRejectedValue(new Error('R2 unavailable'))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const result = await vfs.readPage('page-123')
+      expect(result).toBeNull()
+    })
+  })
+
+  describe('Graceful degradation - writePage() with tier failures', () => {
+    it('should fall back to R2 when DO fails for warm tier', async () => {
+      mockDOStorage.put.mockRejectedValue(new Error('DO unavailable'))
+      mockR2Layer.put.mockResolvedValue(undefined)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const data = new Uint8Array([1, 2, 3])
+      await vfs.writePage('page-456', data, { tier: 'warm' })
+
+      expect(mockDOStorage.put).toHaveBeenCalled()
+      expect(mockR2Layer.put).toHaveBeenCalled()
+    })
+
+    it('should throw when all tiers fail for write', async () => {
+      mockDOStorage.put.mockRejectedValue(new Error('DO unavailable'))
+      mockR2Layer.put.mockRejectedValue(new Error('R2 unavailable'))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const data = new Uint8Array([1, 2, 3])
+      await expect(vfs.writePage('page-456', data)).rejects.toThrow('Failed to write to any tier')
+    })
+  })
+
+  describe('Graceful degradation - deletePage() with tier failures', () => {
+    it('should report partial success when some tiers fail', async () => {
+      mockCacheLayer.delete.mockResolvedValue(true)
+      mockDOStorage.delete.mockRejectedValue(new Error('DO unavailable'))
+      mockR2Layer.delete.mockResolvedValue(undefined)
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      const result = await vfs.deletePage('page-789')
+      expect(result.success).toBe(false)
+      expect(result.failedTiers).toContain('warm')
+    })
+  })
+
+  describe('Graceful degradation - tier health tracking', () => {
+    it('should track tier health status', async () => {
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      await vfs.readPage('page-123')
+
+      const health = vfs.getTierHealth()
+      expect(health.hot).toBe('degraded')
+      expect(health.warm).toBe('healthy')
+    })
+
+    it('should mark tier as unavailable after threshold failures', async () => {
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+        failureThreshold: 3,
+      }
+      const vfs = createTieredVFS(config)
+
+      // Trigger 3 failures
+      await vfs.readPage('page-1')
+      await vfs.readPage('page-2')
+      await vfs.readPage('page-3')
+
+      const health = vfs.getTierHealth()
+      expect(health.hot).toBe('unavailable')
+    })
+
+    it('should skip unavailable tiers', async () => {
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+        failureThreshold: 2,
+      }
+      const vfs = createTieredVFS(config)
+
+      // Mark hot tier as unavailable
+      await vfs.readPage('page-1')
+      await vfs.readPage('page-2')
+
+      vi.clearAllMocks()
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      // This should skip the cache layer entirely
+      await vfs.readPage('page-3')
+      expect(mockCacheLayer.get).not.toHaveBeenCalled()
+      expect(mockDOStorage.get).toHaveBeenCalled()
+    })
+  })
+
+  describe('Graceful degradation - tier failure callbacks', () => {
+    it('should invoke failure callbacks', async () => {
+      const callback = vi.fn()
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+      vfs.onTierFailure(callback)
+
+      await vfs.readPage('page-123')
+
+      expect(callback).toHaveBeenCalledWith(
+        expect.objectContaining({
+          tier: 'hot',
+          operation: 'read',
+          error: 'Cache unavailable',
+        })
+      )
+    })
+  })
+
+  describe('Graceful degradation - tier health reset', () => {
+    it('should allow manual tier health reset', async () => {
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+        failureThreshold: 1,
+      }
+      const vfs = createTieredVFS(config)
+
+      await vfs.readPage('page-123')
+      expect(vfs.getTierHealth().hot).toBe('unavailable')
+
+      vfs.resetTierHealth('hot')
+      expect(vfs.getTierHealth().hot).toBe('healthy')
+    })
+
+    it('should allow resetting all tier health', async () => {
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockRejectedValue(new Error('DO unavailable'))
+      mockR2Layer.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+        failureThreshold: 1,
+      }
+      const vfs = createTieredVFS(config)
+
+      await vfs.readPage('page-123')
+      expect(vfs.getTierHealth().hot).toBe('unavailable')
+      expect(vfs.getTierHealth().warm).toBe('unavailable')
+
+      vfs.resetAllTierHealth()
+      expect(vfs.getTierHealth().hot).toBe('healthy')
+      expect(vfs.getTierHealth().warm).toBe('healthy')
+      expect(vfs.getTierHealth().cold).toBe('healthy')
+    })
+  })
+
+  describe('Graceful degradation - configuration options', () => {
+    it('should support custom failure threshold', () => {
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+        failureThreshold: 5,
+      }
+      expect(config.failureThreshold).toBe(5)
+    })
+
+    it('should support custom recovery time', () => {
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+        recoveryTimeMs: 60000,
+      }
+      expect(config.recoveryTimeMs).toBe(60000)
+    })
+
+    it('should limit failure history size', async () => {
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+        maxFailureHistory: 3,
+        failureThreshold: 10, // High threshold so tier stays healthy/degraded
+      }
+      const vfs = createTieredVFS(config)
+
+      // Trigger 5 failures
+      for (let i = 0; i < 5; i++) {
+        await vfs.readPage(`page-${i}`)
+      }
+
+      const health = vfs.getTierHealth()
+      expect(health.recentFailures.length).toBe(3)
+      // Should have the last 3 failures
+      expect(health.recentFailures.map(f => f.key)).toEqual(['page-2', 'page-3', 'page-4'])
+    })
+  })
+
+  describe('Graceful degradation - stats include tier health', () => {
+    it('should include tier health in stats', async () => {
+      mockCacheLayer.get.mockRejectedValue(new Error('Cache unavailable'))
+      mockDOStorage.get.mockResolvedValue(new Uint8Array([1]))
+
+      const config: TieredVFSConfig = {
+        cacheLayer: mockCacheLayer as unknown as CacheLayer,
+        doStorage: mockDOStorage as unknown as DurableObjectStorage,
+        r2Layer: mockR2Layer as unknown as R2StorageLayer,
+        pageSize: 8192,
+      }
+      const vfs = createTieredVFS(config)
+
+      await vfs.readPage('page-123')
+
+      const stats = vfs.getStats()
+      expect(stats.tierHealth).toBeDefined()
+      expect(stats.tierHealth.hot).toBe('degraded')
+      expect(stats.tierHealth.failureCounts.hot).toBe(1)
+    })
+  })
+})