@dotdo/postgres 0.1.0

package/src/observability/storage-tier-metrics.ts

@@ -0,0 +1,570 @@
+/**
+ * Storage Tier Metrics
+ *
+ * Tracks metrics for the tiered storage architecture:
+ * - HOT: In-DO SQLite Blobs (synchronous, fastest)
+ * - WARM: Cloudflare Cache (free caching layer)
+ * - COLD: R2 object storage (archived/bulk data)
+ */
+
+import type { MetricsCollector } from './types'
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Storage tier names
+ */
+export type StorageTier = 'hot' | 'warm' | 'cold'
+
+/**
+ * Storage operation types
+ */
+export type StorageOperation = 'read' | 'write' | 'delete' | 'list' | 'promote' | 'demote'
+
+/**
+ * Tier transition direction
+ */
+export type TierTransition = 'promote' | 'demote'
+
+/**
+ * Storage operation result
+ */
+export interface StorageOperationResult {
+  /** Which tier served the request */
+  tier: StorageTier
+  /** Whether the operation was a hit (for reads) */
+  hit: boolean
+  /** Operation duration in milliseconds */
+  durationMs: number
+  /** Data size in bytes */
+  bytes?: number
+  /** Whether data was promoted to a faster tier */
+  promoted?: boolean
+}
+
+/**
+ * Tier statistics
+ */
+export interface TierStats {
+  /** Tier name */
+  tier: StorageTier
+  /** Total operations */
+  totalOperations: number
+  /** Read hits */
+  readHits: number
+  /** Read misses */
+  readMisses: number
+  /** Hit ratio (0-1) */
+  hitRatio: number
+  /** Total bytes read */
+  bytesRead: number
+  /** Total bytes written */
+  bytesWritten: number
+  /** Average read latency */
+  avgReadLatencyMs: number
+  /** Average write latency */
+  avgWriteLatencyMs: number
+  /** P95 read latency */
+  p95ReadLatencyMs: number
+  /** Current storage usage in bytes */
+  storageUsageBytes: number
+}
+
+/**
+ * Tier transition statistics
+ */
+export interface TierTransitionStats {
+  /** Promotions from this tier */
+  promotions: number
+  /** Demotions to this tier */
+  demotions: number
+  /** Total bytes promoted */
+  bytesPromoted: number
+  /** Total bytes demoted */
+  bytesDemoted: number
+  /** Average promotion latency */
+  avgPromotionLatencyMs: number
+  /** Average demotion latency */
+  avgDemotionLatencyMs: number
+}
+
+/**
+ * Overall tiering efficiency
+ */
+export interface TieringEfficiency {
+  /** Overall hit ratio across all tiers */
+  overallHitRatio: number
+  /** Percentage of requests served from HOT tier */
+  hotTierPercentage: number
+  /** Percentage of requests served from WARM tier */
+  warmTierPercentage: number
+  /** Percentage of requests served from COLD tier */
+  coldTierPercentage: number
+  /** Estimated cost savings from tiering */
+  estimatedCostSavings: number
+  /** Efficiency score (0-100) */
+  efficiencyScore: number
+}
+
+/**
+ * Storage tier metrics configuration
+ */
+export interface StorageTierMetricsConfig {
+  /** Metrics collector for integration */
+  metricsCollector?: MetricsCollector
+  /** Maximum latency samples to keep per tier */
+  maxLatencySamples?: number
+  /** Cost per GB for HOT tier (DO storage) */
+  hotCostPerGB?: number
+  /** Cost per GB for WARM tier (free) */
+  warmCostPerGB?: number
+  /** Cost per GB for COLD tier (R2) */
+  coldCostPerGB?: number
+}
+
+/**
+ * Storage tier metrics collector interface
+ */
+export interface StorageTierMetricsCollector {
+  /** Record a storage operation */
+  recordOperation(
+    tier: StorageTier,
+    operation: StorageOperation,
+    result: Omit<StorageOperationResult, 'tier'>
+  ): void
+  /** Record a tier transition */
+  recordTransition(
+    fromTier: StorageTier,
+    toTier: StorageTier,
+    bytes: number,
+    durationMs: number
+  ): void
+  /** Update storage usage for a tier */
+  setStorageUsage(tier: StorageTier, bytes: number): void
+  /** Get statistics for a specific tier */
+  getTierStats(tier: StorageTier): TierStats
+  /** Get statistics for all tiers */
+  getAllTierStats(): Record<StorageTier, TierStats>
+  /** Get tier transition statistics */
+  getTransitionStats(): Record<StorageTier, TierTransitionStats>
+  /** Calculate tiering efficiency */
+  getEfficiency(): TieringEfficiency
+  /** Get recommendation for tier configuration */
+  getRecommendation(): TierRecommendation
+  /** Reset all statistics */
+  reset(): void
+}
+
+/**
+ * Tier configuration recommendation
+ */
+export interface TierRecommendation {
+  /** Recommended action */
+  action: 'none' | 'increase_hot' | 'increase_warm' | 'adjust_promotion_policy'
+  /** Reason for recommendation */
+  reason: string
+  /** Current efficiency score */
+  currentEfficiency: number
+  /** Potential efficiency improvement */
+  potentialImprovement: number
+}
+
+// ============================================================================
+// Implementation
+// ============================================================================
+
+/**
+ * Calculate percentile from sorted array
+ */
+function calculatePercentile(sortedValues: number[], percentile: number): number {
+  if (sortedValues.length === 0) return 0
+  const index = Math.ceil((percentile / 100) * sortedValues.length) - 1
+  return sortedValues[Math.max(0, index)] ?? 0
+}
+
+/**
+ * Tier data for tracking
+ */
+interface TierData {
+  totalOperations: number
+  readHits: number
+  readMisses: number
+  bytesRead: number
+  bytesWritten: number
+  readLatencies: number[]
+  writeLatencies: number[]
+  storageUsageBytes: number
+}
+
+/**
+ * Transition data for tracking
+ */
+interface TransitionData {
+  promotions: number
+  demotions: number
+  bytesPromoted: number
+  bytesDemoted: number
+  promotionLatencies: number[]
+  demotionLatencies: number[]
+}
+
+/**
+ * Storage tier metrics collector implementation
+ */
+export class StorageTierMetricsCollectorImpl implements StorageTierMetricsCollector {
+  private config: Required<StorageTierMetricsConfig>
+  private tierData: Map<StorageTier, TierData>
+  private transitionData: Map<StorageTier, TransitionData>
+
+  constructor(config?: StorageTierMetricsConfig) {
+    this.config = {
+      metricsCollector: config?.metricsCollector as MetricsCollector,
+      maxLatencySamples: config?.maxLatencySamples ?? 1000,
+      hotCostPerGB: config?.hotCostPerGB ?? 0.20, // DO storage $0.20/GB-month
+      warmCostPerGB: config?.warmCostPerGB ?? 0, // Cache is free
+      coldCostPerGB: config?.coldCostPerGB ?? 0.015, // R2 $0.015/GB-month
+    }
+
+    this.tierData = new Map()
+    this.transitionData = new Map()
+
+    // Initialize data for all tiers
+    for (const tier of ['hot', 'warm', 'cold'] as StorageTier[]) {
+      this.tierData.set(tier, this.createEmptyTierData())
+      this.transitionData.set(tier, this.createEmptyTransitionData())
+    }
+  }
+
+  private createEmptyTierData(): TierData {
+    return {
+      totalOperations: 0,
+      readHits: 0,
+      readMisses: 0,
+      bytesRead: 0,
+      bytesWritten: 0,
+      readLatencies: [],
+      writeLatencies: [],
+      storageUsageBytes: 0,
+    }
+  }
+
+  private createEmptyTransitionData(): TransitionData {
+    return {
+      promotions: 0,
+      demotions: 0,
+      bytesPromoted: 0,
+      bytesDemoted: 0,
+      promotionLatencies: [],
+      demotionLatencies: [],
+    }
+  }
+
+  recordOperation(
+    tier: StorageTier,
+    operation: StorageOperation,
+    result: Omit<StorageOperationResult, 'tier'>
+  ): void {
+    const data = this.tierData.get(tier)!
+
+    data.totalOperations++
+
+    if (operation === 'read' || operation === 'list') {
+      if (result.hit) {
+        data.readHits++
+      } else {
+        data.readMisses++
+      }
+      if (result.bytes) {
+        data.bytesRead += result.bytes
+      }
+      data.readLatencies.push(result.durationMs)
+      if (data.readLatencies.length > this.config.maxLatencySamples) {
+        data.readLatencies.shift()
+      }
+    } else if (operation === 'write') {
+      if (result.bytes) {
+        data.bytesWritten += result.bytes
+      }
+      data.writeLatencies.push(result.durationMs)
+      if (data.writeLatencies.length > this.config.maxLatencySamples) {
+        data.writeLatencies.shift()
+      }
+    }
+
+    // Record metrics
+    if (this.config.metricsCollector) {
+      this.config.metricsCollector.increment(`postgres.tier.${tier}.operations`, 1, {
+        operation,
+      })
+      if (result.hit !== undefined) {
+        if (result.hit) {
+          this.config.metricsCollector.increment(`postgres.tier.${tier}.hits`, 1)
+        } else {
+          this.config.metricsCollector.increment(`postgres.tier.${tier}.misses`, 1)
+        }
+      }
+      this.config.metricsCollector.histogram(
+        `postgres.tier.${tier}.latency_ms`,
+        result.durationMs,
+        { operation }
+      )
+      if (result.bytes) {
+        this.config.metricsCollector.histogram(`postgres.tier.${tier}.bytes`, result.bytes, {
+          operation,
+        })
+      }
+    }
+  }
+
+  recordTransition(
+    fromTier: StorageTier,
+    toTier: StorageTier,
+    bytes: number,
+    durationMs: number
+  ): void {
+    const isPromotion = this.getTierPriority(toTier) > this.getTierPriority(fromTier)
+
+    // Update source tier
+    const fromData = this.transitionData.get(fromTier)!
+    if (isPromotion) {
+      fromData.promotions++
+      fromData.bytesPromoted += bytes
+      fromData.promotionLatencies.push(durationMs)
+      if (fromData.promotionLatencies.length > this.config.maxLatencySamples) {
+        fromData.promotionLatencies.shift()
+      }
+    } else {
+      fromData.demotions++
+      fromData.bytesDemoted += bytes
+      fromData.demotionLatencies.push(durationMs)
+      if (fromData.demotionLatencies.length > this.config.maxLatencySamples) {
+        fromData.demotionLatencies.shift()
+      }
+    }
+
+    // Record metrics
+    if (this.config.metricsCollector) {
+      const transitionType = isPromotion ? 'promotion' : 'demotion'
+      this.config.metricsCollector.increment(`postgres.tier.transitions`, 1, {
+        from: fromTier,
+        to: toTier,
+        type: transitionType,
+      })
+      this.config.metricsCollector.histogram(`postgres.tier.transition_bytes`, bytes, {
+        from: fromTier,
+        to: toTier,
+      })
+      this.config.metricsCollector.histogram(`postgres.tier.transition_latency_ms`, durationMs, {
+        from: fromTier,
+        to: toTier,
+      })
+    }
+  }
+
+  setStorageUsage(tier: StorageTier, bytes: number): void {
+    const data = this.tierData.get(tier)!
+    data.storageUsageBytes = bytes
+
+    if (this.config.metricsCollector) {
+      this.config.metricsCollector.gauge(`postgres.tier.${tier}.bytes`, bytes)
+    }
+  }
+
+  getTierStats(tier: StorageTier): TierStats {
+    const data = this.tierData.get(tier)!
+    const sortedReadLatencies = [...data.readLatencies].sort((a, b) => a - b)
+
+    const totalReads = data.readHits + data.readMisses
+    const hitRatio = totalReads > 0 ? data.readHits / totalReads : 0
+
+    const avgReadLatencyMs =
+      data.readLatencies.length > 0
+        ? data.readLatencies.reduce((sum, v) => sum + v, 0) / data.readLatencies.length
+        : 0
+
+    const avgWriteLatencyMs =
+      data.writeLatencies.length > 0
+        ? data.writeLatencies.reduce((sum, v) => sum + v, 0) / data.writeLatencies.length
+        : 0
+
+    return {
+      tier,
+      totalOperations: data.totalOperations,
+      readHits: data.readHits,
+      readMisses: data.readMisses,
+      hitRatio,
+      bytesRead: data.bytesRead,
+      bytesWritten: data.bytesWritten,
+      avgReadLatencyMs,
+      avgWriteLatencyMs,
+      p95ReadLatencyMs: calculatePercentile(sortedReadLatencies, 95),
+      storageUsageBytes: data.storageUsageBytes,
+    }
+  }
+
+  getAllTierStats(): Record<StorageTier, TierStats> {
+    return {
+      hot: this.getTierStats('hot'),
+      warm: this.getTierStats('warm'),
+      cold: this.getTierStats('cold'),
+    }
+  }
+
+  getTransitionStats(): Record<StorageTier, TierTransitionStats> {
+    const result: Record<StorageTier, TierTransitionStats> = {} as Record<
+      StorageTier,
+      TierTransitionStats
+    >
+
+    for (const tier of ['hot', 'warm', 'cold'] as StorageTier[]) {
+      const data = this.transitionData.get(tier)!
+
+      const avgPromotionLatencyMs =
+        data.promotionLatencies.length > 0
+          ? data.promotionLatencies.reduce((sum, v) => sum + v, 0) / data.promotionLatencies.length
+          : 0
+
+      const avgDemotionLatencyMs =
+        data.demotionLatencies.length > 0
+          ? data.demotionLatencies.reduce((sum, v) => sum + v, 0) / data.demotionLatencies.length
+          : 0
+
+      result[tier] = {
+        promotions: data.promotions,
+        demotions: data.demotions,
+        bytesPromoted: data.bytesPromoted,
+        bytesDemoted: data.bytesDemoted,
+        avgPromotionLatencyMs,
+        avgDemotionLatencyMs,
+      }
+    }
+
+    return result
+  }
+
+  getEfficiency(): TieringEfficiency {
+    const allStats = this.getAllTierStats()
+
+    const totalHits =
+      allStats.hot.readHits + allStats.warm.readHits + allStats.cold.readHits
+    const totalMisses =
+      allStats.hot.readMisses + allStats.warm.readMisses + allStats.cold.readMisses
+    const totalReads = totalHits + totalMisses
+
+    const overallHitRatio = totalReads > 0 ? totalHits / totalReads : 0
+
+    // Calculate percentage served by each tier
+    const totalOperations =
+      allStats.hot.totalOperations +
+      allStats.warm.totalOperations +
+      allStats.cold.totalOperations
+
+    const hotTierPercentage =
+      totalOperations > 0 ? (allStats.hot.totalOperations / totalOperations) * 100 : 0
+    const warmTierPercentage =
+      totalOperations > 0 ? (allStats.warm.totalOperations / totalOperations) * 100 : 0
+    const coldTierPercentage =
+      totalOperations > 0 ? (allStats.cold.totalOperations / totalOperations) * 100 : 0
+
+    // Estimate cost savings from tiering
+    // If all data was in HOT tier vs. current tiered approach
+    const totalStorageGB =
+      (allStats.hot.storageUsageBytes +
+        allStats.warm.storageUsageBytes +
+        allStats.cold.storageUsageBytes) /
+      (1024 * 1024 * 1024)
+
+    const allHotCost = totalStorageGB * this.config.hotCostPerGB
+    const currentCost =
+      (allStats.hot.storageUsageBytes / (1024 * 1024 * 1024)) * this.config.hotCostPerGB +
+      (allStats.warm.storageUsageBytes / (1024 * 1024 * 1024)) * this.config.warmCostPerGB +
+      (allStats.cold.storageUsageBytes / (1024 * 1024 * 1024)) * this.config.coldCostPerGB
+
+    const estimatedCostSavings = Math.max(0, allHotCost - currentCost)
+
+    // Calculate efficiency score (0-100)
+    // Higher is better: balances hit ratio, tier distribution, and cost savings
+    const hitRatioScore = overallHitRatio * 40 // Max 40 points
+    const tierDistributionScore =
+      (hotTierPercentage * 0.5 + warmTierPercentage * 0.35 + coldTierPercentage * 0.15) * 0.4 // Max 40 points (weighted toward faster tiers)
+    const costScore = Math.min(20, (estimatedCostSavings / 10) * 20) // Max 20 points
+
+    const efficiencyScore = Math.min(100, hitRatioScore + tierDistributionScore + costScore)
+
+    return {
+      overallHitRatio,
+      hotTierPercentage,
+      warmTierPercentage,
+      coldTierPercentage,
+      estimatedCostSavings,
+      efficiencyScore,
+    }
+  }
+
+  getRecommendation(): TierRecommendation {
+    const efficiency = this.getEfficiency()
+    const allStats = this.getAllTierStats()
+
+    let action: TierRecommendation['action'] = 'none'
+    let reason = 'Tier configuration is optimal'
+    let potentialImprovement = 0
+
+    // Check for low hit ratio
+    if (efficiency.overallHitRatio < 0.5) {
+      action = 'adjust_promotion_policy'
+      reason = 'Low hit ratio suggests frequently accessed data is not being promoted to faster tiers'
+      potentialImprovement = 15
+    }
+
+    // Check if HOT tier is underutilized
+    if (efficiency.hotTierPercentage < 20 && allStats.hot.hitRatio < 0.8) {
+      action = 'increase_hot'
+      reason = 'HOT tier is underutilized. Consider promoting more frequently accessed data.'
+      potentialImprovement = 10
+    }
+
+    // Check if WARM tier could be used more
+    if (efficiency.warmTierPercentage < 30 && allStats.cold.totalOperations > 0) {
+      action = 'increase_warm'
+      reason =
+        'WARM tier (free cache) is underutilized. Consider caching more COLD tier data.'
+      potentialImprovement = 20
+    }
+
+    return {
+      action,
+      reason,
+      currentEfficiency: efficiency.efficiencyScore,
+      potentialImprovement,
+    }
+  }
+
+  reset(): void {
+    for (const tier of ['hot', 'warm', 'cold'] as StorageTier[]) {
+      this.tierData.set(tier, this.createEmptyTierData())
+      this.transitionData.set(tier, this.createEmptyTransitionData())
+    }
+  }
+
+  private getTierPriority(tier: StorageTier): number {
+    switch (tier) {
+      case 'hot':
+        return 3
+      case 'warm':
+        return 2
+      case 'cold':
+        return 1
+    }
+  }
+}
+
+/**
+ * Create a storage tier metrics collector
+ */
+export function createStorageTierMetricsCollector(
+  config?: StorageTierMetricsConfig
+): StorageTierMetricsCollector {
+  return new StorageTierMetricsCollectorImpl(config)
+}