@dotdo/postgres 0.1.0

package/src/iceberg/analytics.ts

@@ -0,0 +1,727 @@
+/**
+ * Iceberg Historical Data Analytics
+ * Task: postgres-no1 - Historical data analytics feature
+ *
+ * Provides historical analytics capabilities for Iceberg tables:
+ * - Change rate analysis (inserts, updates, deletes over time)
+ * - Row count trends across snapshots
+ * - Data growth metrics and projections
+ * - Snapshot comparison and diff analysis
+ *
+ * @example
+ * ```typescript
+ * const analytics = createHistoricalAnalytics({
+ *   bucket: env.ICEBERG_BUCKET,
+ *   catalog,
+ * })
+ *
+ * // Get change rates over a time period
+ * const changeRate = await analytics.getChangeRate(
+ *   tableId,
+ *   new Date('2024-01-01'),
+ *   new Date('2024-01-31')
+ * )
+ *
+ * // Get data growth metrics
+ * const growth = await analytics.getGrowthMetrics(tableId)
+ *
+ * // Compare two snapshots
+ * const comparison = await analytics.getSnapshotComparison(
+ *   tableId,
+ *   snapshotId1,
+ *   snapshotId2
+ * )
+ * ```
+ */
+
+import type { IcebergDataFile } from './types'
+import type { IcebergCatalog, TableIdentifier } from './catalog'
+import {
+  IcebergTimeTravelReader,
+  parseTimestamp,
+  formatTimestamp,
+  type TimestampSpec,
+} from './time-travel'
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Change rate analysis result
+ */
+export interface ChangeRateAnalysis {
+  /** Start timestamp of analysis period */
+  startTimestampMs: number
+  /** End timestamp of analysis period */
+  endTimestampMs: number
+  /** Duration of analysis period in milliseconds */
+  durationMs: number
+  /** Total records added */
+  totalAdded: number
+  /** Total records deleted */
+  totalDeleted: number
+  /** Net change in records */
+  netChange: number
+  /** Average records added per day */
+  addedPerDay: number
+  /** Average records deleted per day */
+  deletedPerDay: number
+  /** Average net change per day */
+  netChangePerDay: number
+  /** Number of snapshots in period */
+  snapshotCount: number
+  /** Snapshots analyzed */
+  snapshots: SnapshotChangeInfo[]
+}
+
+/**
+ * Information about changes in a specific snapshot
+ */
+export interface SnapshotChangeInfo {
+  /** Snapshot ID */
+  snapshotId: number
+  /** Snapshot timestamp */
+  timestampMs: number
+  /** Timestamp formatted as ISO string */
+  timestamp: string
+  /** Records added in this snapshot */
+  recordsAdded: number
+  /** Records deleted in this snapshot */
+  recordsDeleted: number
+  /** Net change in this snapshot */
+  netChange: number
+  /** Total records after this snapshot */
+  totalRecords: number
+  /** Operation type */
+  operation: string
+}
+
+/**
+ * Data growth metrics
+ */
+export interface GrowthMetrics {
+  /** Current total records */
+  currentRecordCount: number
+  /** Current total data size in bytes */
+  currentDataSizeBytes: number
+  /** Current number of data files */
+  currentFileCount: number
+  /** Oldest snapshot timestamp */
+  oldestSnapshotMs: number | null
+  /** Newest snapshot timestamp */
+  newestSnapshotMs: number | null
+  /** Total time span in milliseconds */
+  timeSpanMs: number
+  /** Average growth rate (records per day) */
+  avgRecordsPerDay: number
+  /** Average data size growth (bytes per day) */
+  avgBytesPerDay: number
+  /** Projected record count in 30 days */
+  projectedRecordsIn30Days: number
+  /** Projected data size in 30 days (bytes) */
+  projectedBytesIn30Days: number
+  /** Growth trend */
+  trend: 'growing' | 'shrinking' | 'stable'
+  /** Historical data points */
+  history: GrowthDataPoint[]
+}
+
+/**
+ * Growth data point at a specific time
+ */
+export interface GrowthDataPoint {
+  /** Snapshot ID */
+  snapshotId: number
+  /** Timestamp */
+  timestampMs: number
+  /** Timestamp formatted */
+  timestamp: string
+  /** Total records at this point */
+  recordCount: number
+  /** Total data size at this point */
+  dataSizeBytes: number
+  /** Number of files at this point */
+  fileCount: number
+}
+
+/**
+ * Snapshot comparison result
+ */
+export interface SnapshotComparison {
+  /** First snapshot info */
+  snapshot1: SnapshotComparisonInfo
+  /** Second snapshot info */
+  snapshot2: SnapshotComparisonInfo
+  /** Time difference in milliseconds */
+  timeDifferenceMs: number
+  /** Record count difference (snapshot2 - snapshot1) */
+  recordCountDelta: number
+  /** Data size difference in bytes (snapshot2 - snapshot1) */
+  dataSizeDelta: number
+  /** File count difference (snapshot2 - snapshot1) */
+  fileCountDelta: number
+  /** Files added between snapshots */
+  filesAdded: IcebergDataFile[]
+  /** Files deleted between snapshots */
+  filesDeleted: IcebergDataFile[]
+  /** Files that exist in both snapshots */
+  filesUnchanged: IcebergDataFile[]
+  /** Summary statistics */
+  summary: {
+    /** Percentage change in record count */
+    recordCountChangePercent: number
+    /** Percentage change in data size */
+    dataSizeChangePercent: number
+    /** Average file size in snapshot 1 */
+    avgFileSizeSnapshot1: number
+    /** Average file size in snapshot 2 */
+    avgFileSizeSnapshot2: number
+  }
+}
+
+/**
+ * Snapshot info for comparison
+ */
+export interface SnapshotComparisonInfo {
+  /** Snapshot ID */
+  snapshotId: number
+  /** Snapshot timestamp */
+  timestampMs: number
+  /** Timestamp formatted */
+  timestamp: string
+  /** Total records */
+  recordCount: number
+  /** Total data size */
+  dataSizeBytes: number
+  /** Number of files */
+  fileCount: number
+  /** Operation type */
+  operation: string
+}
+
+/**
+ * Historical analytics configuration
+ */
+export interface HistoricalAnalyticsConfig {
+  /** R2 bucket for reading data */
+  bucket: R2Bucket
+  /** Iceberg catalog instance */
+  catalog: IcebergCatalog
+  /** Default retention window for analytics (default: 90 days) */
+  retentionWindowMs?: number
+  /** Enable caching of analysis results (default: true) */
+  enableCache?: boolean
+  /** Cache TTL in milliseconds (default: 5 minutes) */
+  cacheTtlMs?: number
+}
+
+/**
+ * Cache entry for analytics results
+ */
+interface CacheEntry<T> {
+  data: T
+  expiresAt: number
+}
+
+// ============================================================================
+// Historical Analytics Class
+// ============================================================================
+
+/**
+ * Historical Analytics for Iceberg Tables
+ *
+ * Provides advanced analytics capabilities for Iceberg time-travel data:
+ * - Change rate analysis
+ * - Growth trend analysis
+ * - Snapshot comparison
+ */
+export class HistoricalAnalytics {
+  private bucket: R2Bucket
+  private catalog: IcebergCatalog
+  private reader: IcebergTimeTravelReader
+  private retentionWindowMs: number
+  private enableCache: boolean
+  private cacheTtlMs: number
+  private cache: Map<string, CacheEntry<unknown>> = new Map()
+
+  constructor(config: HistoricalAnalyticsConfig) {
+    this.bucket = config.bucket
+    this.catalog = config.catalog
+    this.retentionWindowMs = config.retentionWindowMs ?? 90 * 24 * 60 * 60 * 1000 // 90 days
+    this.enableCache = config.enableCache ?? true
+    this.cacheTtlMs = config.cacheTtlMs ?? 5 * 60 * 1000 // 5 minutes
+
+    this.reader = new IcebergTimeTravelReader({
+      bucket: this.bucket,
+      retentionWindowMs: this.retentionWindowMs,
+    })
+  }
+
+  // ========== Change Rate Analysis ==========
+
+  /**
+   * Get change rate analysis for a table over a time period
+   *
+   * Analyzes insert, update, and delete rates between two timestamps.
+   *
+   * @param table - Table identifier
+   * @param startTime - Start timestamp
+   * @param endTime - End timestamp
+   * @returns Change rate analysis
+   */
+  async getChangeRate(
+    table: TableIdentifier,
+    startTime: TimestampSpec,
+    endTime: TimestampSpec
+  ): Promise<ChangeRateAnalysis> {
+    const cacheKey = this.getCacheKey('changeRate', table, { startTime, endTime })
+    const cached = this.getFromCache<ChangeRateAnalysis>(cacheKey)
+    if (cached) {
+      return cached
+    }
+
+    const metadata = await this.catalog.loadTable(table)
+    const start = parseTimestamp(startTime)
+    const end = parseTimestamp(endTime)
+
+    // Get all snapshots in the time range
+    const snapshotsInRange = await this.reader.findSnapshotsInRange(
+      metadata,
+      start.timestampMs,
+      end.timestampMs
+    )
+
+    if (snapshotsInRange.length === 0) {
+      const result: ChangeRateAnalysis = {
+        startTimestampMs: start.timestampMs,
+        endTimestampMs: end.timestampMs,
+        durationMs: end.timestampMs - start.timestampMs,
+        totalAdded: 0,
+        totalDeleted: 0,
+        netChange: 0,
+        addedPerDay: 0,
+        deletedPerDay: 0,
+        netChangePerDay: 0,
+        snapshotCount: 0,
+        snapshots: [],
+      }
+      this.setInCache(cacheKey, result)
+      return result
+    }
+
+    // Sort snapshots chronologically
+    snapshotsInRange.sort((a, b) => a['timestamp-ms'] - b['timestamp-ms'])
+
+    // Analyze each snapshot
+    const snapshotChanges: SnapshotChangeInfo[] = []
+    let totalAdded = 0
+    let totalDeleted = 0
+
+    for (const snapshot of snapshotsInRange) {
+      const summary = snapshot.summary
+      const recordsAdded = parseInt(summary['added-records'] ?? '0', 10)
+      const recordsDeleted = parseInt(summary['deleted-records'] ?? '0', 10)
+      const totalRecords = parseInt(summary['total-records'] ?? '0', 10)
+
+      totalAdded += recordsAdded
+      totalDeleted += recordsDeleted
+
+      snapshotChanges.push({
+        snapshotId: snapshot['snapshot-id'],
+        timestampMs: snapshot['timestamp-ms'],
+        timestamp: formatTimestamp(snapshot['timestamp-ms']),
+        recordsAdded,
+        recordsDeleted,
+        netChange: recordsAdded - recordsDeleted,
+        totalRecords,
+        operation: summary.operation,
+      })
+    }
+
+    // Calculate rates
+    const durationMs = end.timestampMs - start.timestampMs
+    const durationDays = durationMs / (24 * 60 * 60 * 1000)
+
+    const result: ChangeRateAnalysis = {
+      startTimestampMs: start.timestampMs,
+      endTimestampMs: end.timestampMs,
+      durationMs,
+      totalAdded,
+      totalDeleted,
+      netChange: totalAdded - totalDeleted,
+      addedPerDay: durationDays > 0 ? totalAdded / durationDays : 0,
+      deletedPerDay: durationDays > 0 ? totalDeleted / durationDays : 0,
+      netChangePerDay: durationDays > 0 ? (totalAdded - totalDeleted) / durationDays : 0,
+      snapshotCount: snapshotsInRange.length,
+      snapshots: snapshotChanges,
+    }
+
+    this.setInCache(cacheKey, result)
+    return result
+  }
+
+  // ========== Growth Metrics ==========
+
+  /**
+   * Get data growth metrics for a table
+   *
+   * Analyzes historical growth patterns and projects future growth.
+   *
+   * @param table - Table identifier
+   * @param samplingInterval - Optional sampling interval in ms (default: 1 day)
+   * @returns Growth metrics
+   */
+  async getGrowthMetrics(
+    table: TableIdentifier,
+    samplingInterval = 24 * 60 * 60 * 1000 // 1 day
+  ): Promise<GrowthMetrics> {
+    const cacheKey = this.getCacheKey('growthMetrics', table, { samplingInterval })
+    const cached = this.getFromCache<GrowthMetrics>(cacheKey)
+    if (cached) {
+      return cached
+    }
+
+    const metadata = await this.catalog.loadTable(table)
+    const snapshots = metadata.snapshots ?? []
+
+    if (snapshots.length === 0) {
+      const result: GrowthMetrics = {
+        currentRecordCount: 0,
+        currentDataSizeBytes: 0,
+        currentFileCount: 0,
+        oldestSnapshotMs: null,
+        newestSnapshotMs: null,
+        timeSpanMs: 0,
+        avgRecordsPerDay: 0,
+        avgBytesPerDay: 0,
+        projectedRecordsIn30Days: 0,
+        projectedBytesIn30Days: 0,
+        trend: 'stable',
+        history: [],
+      }
+      this.setInCache(cacheKey, result)
+      return result
+    }
+
+    // Sort snapshots chronologically
+    const sortedSnapshots = [...snapshots].sort(
+      (a, b) => a['timestamp-ms'] - b['timestamp-ms']
+    )
+
+    const oldestSnapshot = sortedSnapshots[0]!
+    const newestSnapshot = sortedSnapshots[sortedSnapshots.length - 1]!
+
+    // Sample snapshots at the given interval
+    const history: GrowthDataPoint[] = []
+    let lastSampledTime = 0
+
+    for (const snapshot of sortedSnapshots) {
+      const snapshotTime = snapshot['timestamp-ms']
+
+      if (snapshotTime - lastSampledTime >= samplingInterval || history.length === 0) {
+        const summary = snapshot.summary
+        const recordCount = parseInt(summary['total-records'] ?? '0', 10)
+        const dataSizeBytes = parseInt(summary['total-files-size'] ?? '0', 10)
+        const fileCount = parseInt(summary['added-data-files'] ?? '0', 10)
+
+        history.push({
+          snapshotId: snapshot['snapshot-id'],
+          timestampMs: snapshotTime,
+          timestamp: formatTimestamp(snapshotTime),
+          recordCount,
+          dataSizeBytes,
+          fileCount,
+        })
+
+        lastSampledTime = snapshotTime
+      }
+    }
+
+    // Ensure we have the latest snapshot in history
+    const latestInHistory = history[history.length - 1]
+    if (latestInHistory && latestInHistory.snapshotId !== newestSnapshot['snapshot-id']) {
+      const summary = newestSnapshot.summary
+      history.push({
+        snapshotId: newestSnapshot['snapshot-id'],
+        timestampMs: newestSnapshot['timestamp-ms'],
+        timestamp: formatTimestamp(newestSnapshot['timestamp-ms']),
+        recordCount: parseInt(summary['total-records'] ?? '0', 10),
+        dataSizeBytes: parseInt(summary['total-files-size'] ?? '0', 10),
+        fileCount: parseInt(summary['added-data-files'] ?? '0', 10),
+      })
+    }
+
+    // Calculate current metrics
+    const currentSummary = newestSnapshot.summary
+    const currentRecordCount = parseInt(currentSummary['total-records'] ?? '0', 10)
+    const currentDataSizeBytes = parseInt(currentSummary['total-files-size'] ?? '0', 10)
+    const currentFileCount = parseInt(currentSummary['added-data-files'] ?? '0', 10)
+
+    // Calculate growth rates
+    const timeSpanMs = newestSnapshot['timestamp-ms'] - oldestSnapshot['timestamp-ms']
+    const timeSpanDays = timeSpanMs / (24 * 60 * 60 * 1000)
+
+    const oldestSummary = oldestSnapshot.summary
+    const oldestRecordCount = parseInt(oldestSummary['total-records'] ?? '0', 10)
+    const oldestDataSizeBytes = parseInt(oldestSummary['total-files-size'] ?? '0', 10)
+
+    const recordGrowth = currentRecordCount - oldestRecordCount
+    const bytesGrowth = currentDataSizeBytes - oldestDataSizeBytes
+
+    const avgRecordsPerDay = timeSpanDays > 0 ? recordGrowth / timeSpanDays : 0
+    const avgBytesPerDay = timeSpanDays > 0 ? bytesGrowth / timeSpanDays : 0
+
+    // Project 30 days into the future
+    const projectedRecordsIn30Days = Math.max(0, currentRecordCount + avgRecordsPerDay * 30)
+    const projectedBytesIn30Days = Math.max(0, currentDataSizeBytes + avgBytesPerDay * 30)
+
+    // Determine trend
+    let trend: 'growing' | 'shrinking' | 'stable' = 'stable'
+    if (avgRecordsPerDay >= 10) {
+      // Growing if adding >= 10 records/day on average
+      trend = 'growing'
+    } else if (avgRecordsPerDay <= -10) {
+      // Shrinking if losing >= 10 records/day on average
+      trend = 'shrinking'
+    }
+
+    const result: GrowthMetrics = {
+      currentRecordCount,
+      currentDataSizeBytes,
+      currentFileCount,
+      oldestSnapshotMs: oldestSnapshot['timestamp-ms'],
+      newestSnapshotMs: newestSnapshot['timestamp-ms'],
+      timeSpanMs,
+      avgRecordsPerDay,
+      avgBytesPerDay,
+      projectedRecordsIn30Days,
+      projectedBytesIn30Days,
+      trend,
+      history,
+    }
+
+    this.setInCache(cacheKey, result)
+    return result
+  }
+
+  // ========== Snapshot Comparison ==========
+
+  /**
+   * Compare two snapshots and return the differences
+   *
+   * Analyzes the changes between two snapshots, including file additions,
+   * deletions, and statistical changes.
+   *
+   * @param table - Table identifier
+   * @param snapshot1Id - First snapshot ID (earlier)
+   * @param snapshot2Id - Second snapshot ID (later)
+   * @returns Snapshot comparison result
+   */
+  async getSnapshotComparison(
+    table: TableIdentifier,
+    snapshot1Id: number,
+    snapshot2Id: number
+  ): Promise<SnapshotComparison> {
+    const cacheKey = this.getCacheKey('snapshotComparison', table, {
+      snapshot1Id,
+      snapshot2Id,
+    })
+    const cached = this.getFromCache<SnapshotComparison>(cacheKey)
+    if (cached) {
+      return cached
+    }
+
+    const metadata = await this.catalog.loadTable(table)
+
+    // Find the snapshots
+    const snapshot1 = metadata.snapshots?.find((s) => s['snapshot-id'] === snapshot1Id)
+    const snapshot2 = metadata.snapshots?.find((s) => s['snapshot-id'] === snapshot2Id)
+
+    if (!snapshot1) {
+      throw new Error(`Snapshot ${snapshot1Id} not found`)
+    }
+    if (!snapshot2) {
+      throw new Error(`Snapshot ${snapshot2Id} not found`)
+    }
+
+    // Get data files for both snapshots
+    const files1 = await this.reader.getDataFilesForSnapshot(metadata, snapshot1)
+    const files2 = await this.reader.getDataFilesForSnapshot(metadata, snapshot2)
+
+    // Create maps for efficient lookup
+    const files1Map = new Map(files1.dataFiles.map((f) => [f['file-path'], f]))
+    const files2Map = new Map(files2.dataFiles.map((f) => [f['file-path'], f]))
+
+    // Find added, deleted, and unchanged files
+    const filesAdded: IcebergDataFile[] = []
+    const filesDeleted: IcebergDataFile[] = []
+    const filesUnchanged: IcebergDataFile[] = []
+
+    // Files in snapshot2 but not in snapshot1 = added
+    for (const [path, file] of files2Map) {
+      if (!files1Map.has(path)) {
+        filesAdded.push(file)
+      } else {
+        filesUnchanged.push(file)
+      }
+    }
+
+    // Files in snapshot1 but not in snapshot2 = deleted
+    for (const [path, file] of files1Map) {
+      if (!files2Map.has(path)) {
+        filesDeleted.push(file)
+      }
+    }
+
+    // Extract summary info
+    const summary1 = snapshot1.summary
+    const summary2 = snapshot2.summary
+
+    const recordCount1 = parseInt(summary1['total-records'] ?? '0', 10)
+    const recordCount2 = parseInt(summary2['total-records'] ?? '0', 10)
+    const dataSize1 = parseInt(summary1['total-files-size'] ?? '0', 10)
+    const dataSize2 = parseInt(summary2['total-files-size'] ?? '0', 10)
+
+    // Calculate deltas
+    const recordCountDelta = recordCount2 - recordCount1
+    const dataSizeDelta = dataSize2 - dataSize1
+    const fileCountDelta = files2.dataFiles.length - files1.dataFiles.length
+    const timeDifferenceMs = snapshot2['timestamp-ms'] - snapshot1['timestamp-ms']
+
+    // Calculate percentage changes
+    const recordCountChangePercent =
+      recordCount1 > 0 ? ((recordCountDelta / recordCount1) * 100) : 0
+    const dataSizeChangePercent =
+      dataSize1 > 0 ? ((dataSizeDelta / dataSize1) * 100) : 0
+
+    // Calculate average file sizes
+    const avgFileSizeSnapshot1 =
+      files1.dataFiles.length > 0
+        ? files1.dataFiles.reduce((sum, f) => sum + f['file-size-in-bytes'], 0) /
+          files1.dataFiles.length
+        : 0
+    const avgFileSizeSnapshot2 =
+      files2.dataFiles.length > 0
+        ? files2.dataFiles.reduce((sum, f) => sum + f['file-size-in-bytes'], 0) /
+          files2.dataFiles.length
+        : 0
+
+    const result: SnapshotComparison = {
+      snapshot1: {
+        snapshotId: snapshot1['snapshot-id'],
+        timestampMs: snapshot1['timestamp-ms'],
+        timestamp: formatTimestamp(snapshot1['timestamp-ms']),
+        recordCount: recordCount1,
+        dataSizeBytes: dataSize1,
+        fileCount: files1.dataFiles.length,
+        operation: summary1.operation,
+      },
+      snapshot2: {
+        snapshotId: snapshot2['snapshot-id'],
+        timestampMs: snapshot2['timestamp-ms'],
+        timestamp: formatTimestamp(snapshot2['timestamp-ms']),
+        recordCount: recordCount2,
+        dataSizeBytes: dataSize2,
+        fileCount: files2.dataFiles.length,
+        operation: summary2.operation,
+      },
+      timeDifferenceMs,
+      recordCountDelta,
+      dataSizeDelta,
+      fileCountDelta,
+      filesAdded,
+      filesDeleted,
+      filesUnchanged,
+      summary: {
+        recordCountChangePercent,
+        dataSizeChangePercent,
+        avgFileSizeSnapshot1,
+        avgFileSizeSnapshot2,
+      },
+    }
+
+    this.setInCache(cacheKey, result)
+    return result
+  }
+
+  // ========== Cache Management ==========
+
+  /**
+   * Generate cache key
+   */
+  private getCacheKey(
+    operation: string,
+    table: TableIdentifier,
+    params: Record<string, unknown>
+  ): string {
+    const namespace = table.namespace.levels.join('.')
+    const paramsStr = JSON.stringify(params)
+    return `${operation}:${namespace}.${table.name}:${paramsStr}`
+  }
+
+  /**
+   * Get value from cache
+   */
+  private getFromCache<T>(key: string): T | null {
+    if (!this.enableCache) {
+      return null
+    }
+
+    const entry = this.cache.get(key)
+    if (!entry) {
+      return null
+    }
+
+    if (entry.expiresAt < Date.now()) {
+      this.cache.delete(key)
+      return null
+    }
+
+    return entry.data as T
+  }
+
+  /**
+   * Set value in cache
+   */
+  private setInCache(key: string, data: unknown): void {
+    if (!this.enableCache) {
+      return
+    }
+
+    this.cache.set(key, {
+      data,
+      expiresAt: Date.now() + this.cacheTtlMs,
+    })
+  }
+
+  /**
+   * Clear the cache
+   */
+  clearCache(): void {
+    this.cache.clear()
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getCacheStats(): { size: number; ttlMs: number } {
+    return {
+      size: this.cache.size,
+      ttlMs: this.cacheTtlMs,
+    }
+  }
+}
+
+// ============================================================================
+// Factory Function
+// ============================================================================
+
+/**
+ * Create a historical analytics instance
+ */
+export function createHistoricalAnalytics(
+  config: HistoricalAnalyticsConfig
+): HistoricalAnalytics {
+  return new HistoricalAnalytics(config)
+}