@dotdo/postgres 0.1.0

package/src/migrations/progress-tracker.ts

@@ -0,0 +1,665 @@
+/**
+ * Migration Progress Tracker
+ *
+ * Tracks migration progress across thousands of Durable Objects.
+ * Provides centralized progress reporting, pause/resume capabilities,
+ * and aggregate statistics.
+ *
+ * @module migrations/progress-tracker
+ */
+
+import { createLogger, LogLevel, type ILogger } from '@dotdo/postgres-shared'
+
+// Re-export MigrationStatus for external use
+export type { MigrationStatus } from './types'
+
+// Module-level logger for ProgressTracker
+// Level is DEBUG because the debug config flag gates whether log() is called at all.
+// The logger just provides structured output - the filtering happens at call site.
+const progressTrackerLogger: ILogger = createLogger({
+  level: LogLevel.DEBUG,
+  prefix: '[ProgressTracker]',
+})
+
+/**
+ * Status of a migration operation across multiple DOs
+ */
+export type MigrationOperationStatus = 'pending' | 'running' | 'paused' | 'completed' | 'failed'
+
+/**
+ * Status of an individual DO's migration
+ */
+export type DOMigrationStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped'
+
+/**
+ * Progress state for a single DO
+ */
+export interface DOProgressState {
+  /** Durable Object ID */
+  doId: string
+  /** Current migration status */
+  status: DOMigrationStatus
+  /** Current migration version being applied */
+  currentVersion: number | null
+  /** Target version to migrate to */
+  targetVersion: number
+  /** Number of migrations completed */
+  migrationsCompleted: number
+  /** Total migrations to apply */
+  totalMigrations: number
+  /** Last error message if failed */
+  error?: string
+  /** When this DO started migrating */
+  startedAt?: string
+  /** When this DO completed/failed */
+  completedAt?: string
+  /** Total execution time in milliseconds */
+  executionTimeMs?: number
+}
+
+/**
+ * Aggregate progress statistics
+ */
+export interface ProgressStats {
+  /** Total number of DOs being migrated */
+  total: number
+  /** Number of DOs that haven't started */
+  pending: number
+  /** Number of DOs currently migrating */
+  running: number
+  /** Number of DOs that completed successfully */
+  completed: number
+  /** Number of DOs that failed */
+  failed: number
+  /** Number of DOs that were skipped */
+  skipped: number
+  /** Overall completion percentage (0-100) */
+  percentComplete: number
+  /** Estimated time remaining in milliseconds (null if not available) */
+  estimatedTimeRemainingMs: number | null
+  /** Average execution time per DO in milliseconds */
+  avgExecutionTimeMs: number
+}
+
+/**
+ * Migration operation record
+ */
+export interface MigrationOperation {
+  /** Unique migration operation ID */
+  migrationId: string
+  /** Target schema version */
+  targetVersion: number
+  /** Overall operation status */
+  status: MigrationOperationStatus
+  /** Total DOs to migrate */
+  totalDOs: number
+  /** When the operation started */
+  startedAt: string
+  /** When the operation completed/failed (if applicable) */
+  completedAt?: string
+  /** User/system that initiated the operation */
+  initiatedBy?: string
+  /** Optional description/reason */
+  description?: string
+  /** Progress statistics */
+  stats: ProgressStats
+}
+
+/**
+ * Storage interface for progress tracking
+ *
+ * Implementations can use Durable Object storage, KV, or other backends
+ */
+export interface ProgressStorage {
+  /**
+   * Store migration operation metadata
+   */
+  setOperation(migrationId: string, operation: MigrationOperation): Promise<void>
+
+  /**
+   * Get migration operation metadata
+   */
+  getOperation(migrationId: string): Promise<MigrationOperation | null>
+
+  /**
+   * Store DO progress state
+   */
+  setDOProgress(migrationId: string, doId: string, progress: DOProgressState): Promise<void>
+
+  /**
+   * Get DO progress state
+   */
+  getDOProgress(migrationId: string, doId: string): Promise<DOProgressState | null>
+
+  /**
+   * Get all DO progress states for a migration
+   */
+  getAllDOProgress(migrationId: string): Promise<DOProgressState[]>
+
+  /**
+   * Get DO progress states by status
+   */
+  getDOProgressByStatus(migrationId: string, status: DOMigrationStatus): Promise<DOProgressState[]>
+
+  /**
+   * Delete migration operation and all associated progress
+   */
+  deleteOperation(migrationId: string): Promise<void>
+
+  /**
+   * List all migration operations
+   */
+  listOperations(): Promise<MigrationOperation[]>
+}
+
+/**
+ * In-memory storage implementation (for testing/single-instance use)
+ */
+export class InMemoryProgressStorage implements ProgressStorage {
+  private operations = new Map<string, MigrationOperation>()
+  private doProgress = new Map<string, Map<string, DOProgressState>>()
+
+  async setOperation(migrationId: string, operation: MigrationOperation): Promise<void> {
+    this.operations.set(migrationId, operation)
+  }
+
+  async getOperation(migrationId: string): Promise<MigrationOperation | null> {
+    return this.operations.get(migrationId) ?? null
+  }
+
+  async setDOProgress(migrationId: string, doId: string, progress: DOProgressState): Promise<void> {
+    if (!this.doProgress.has(migrationId)) {
+      this.doProgress.set(migrationId, new Map())
+    }
+    this.doProgress.get(migrationId)!.set(doId, progress)
+  }
+
+  async getDOProgress(migrationId: string, doId: string): Promise<DOProgressState | null> {
+    return this.doProgress.get(migrationId)?.get(doId) ?? null
+  }
+
+  async getAllDOProgress(migrationId: string): Promise<DOProgressState[]> {
+    const progressMap = this.doProgress.get(migrationId)
+    return progressMap ? Array.from(progressMap.values()) : []
+  }
+
+  async getDOProgressByStatus(
+    migrationId: string,
+    status: DOMigrationStatus
+  ): Promise<DOProgressState[]> {
+    const allProgress = await this.getAllDOProgress(migrationId)
+    return allProgress.filter((p) => p.status === status)
+  }
+
+  async deleteOperation(migrationId: string): Promise<void> {
+    this.operations.delete(migrationId)
+    this.doProgress.delete(migrationId)
+  }
+
+  async listOperations(): Promise<MigrationOperation[]> {
+    return Array.from(this.operations.values())
+  }
+
+  /**
+   * Clear all data (useful for testing)
+   */
+  clear(): void {
+    this.operations.clear()
+    this.doProgress.clear()
+  }
+}
+
+/**
+ * Configuration for the progress tracker
+ */
+export interface ProgressTrackerConfig {
+  /** Storage backend for progress data */
+  storage: ProgressStorage
+  /** Enable debug logging */
+  debug?: boolean
+}
+
+/**
+ * Migration Progress Tracker
+ *
+ * Tracks migration progress across many Durable Objects, providing
+ * aggregate statistics and pause/resume capabilities.
+ *
+ * @example
+ * ```typescript
+ * const tracker = new MigrationProgressTracker({
+ *   storage: new InMemoryProgressStorage(),
+ * })
+ *
+ * // Start tracking a migration
+ * await tracker.startMigration('migration-2024-01', [
+ *   'do-1', 'do-2', 'do-3'
+ * ], 5)
+ *
+ * // Report progress from individual DOs
+ * await tracker.reportProgress('migration-2024-01', {
+ *   doId: 'do-1',
+ *   status: 'running',
+ *   currentVersion: 3,
+ *   targetVersion: 5,
+ *   migrationsCompleted: 3,
+ *   totalMigrations: 5,
+ * })
+ *
+ * // Get overall progress
+ * const progress = await tracker.getProgress('migration-2024-01')
+ * console.log(`${progress.stats.percentComplete}% complete`)
+ * ```
+ */
+export class MigrationProgressTracker {
+  private storage: ProgressStorage
+  private debug: boolean
+
+  constructor(config: ProgressTrackerConfig) {
+    this.storage = config.storage
+    this.debug = config.debug ?? false
+  }
+
+  /**
+   * Log debug message
+   */
+  private log(message: string, ...args: unknown[]): void {
+    if (this.debug) {
+      progressTrackerLogger.debug(message, args.length > 0 ? { args } : undefined)
+    }
+  }
+
+  /**
+   * Calculate progress statistics
+   */
+  private async calculateStats(migrationId: string): Promise<ProgressStats> {
+    const allProgress = await this.storage.getAllDOProgress(migrationId)
+    const total = allProgress.length
+
+    if (total === 0) {
+      return {
+        total: 0,
+        pending: 0,
+        running: 0,
+        completed: 0,
+        failed: 0,
+        skipped: 0,
+        percentComplete: 0,
+        estimatedTimeRemainingMs: null,
+        avgExecutionTimeMs: 0,
+      }
+    }
+
+    const pending = allProgress.filter((p) => p.status === 'pending').length
+    const running = allProgress.filter((p) => p.status === 'running').length
+    const completed = allProgress.filter((p) => p.status === 'completed').length
+    const failed = allProgress.filter((p) => p.status === 'failed').length
+    const skipped = allProgress.filter((p) => p.status === 'skipped').length
+
+    const percentComplete = Math.round(((completed + failed + skipped) / total) * 100)
+
+    // Calculate average execution time from completed DOs
+    const completedDOs = allProgress.filter(
+      (p) => (p.status === 'completed' || p.status === 'failed') && p.executionTimeMs
+    )
+    const avgExecutionTimeMs =
+      completedDOs.length > 0
+        ? completedDOs.reduce((sum, p) => sum + (p.executionTimeMs ?? 0), 0) / completedDOs.length
+        : 0
+
+    // Estimate remaining time
+    let estimatedTimeRemainingMs: number | null = null
+    if (avgExecutionTimeMs > 0 && pending + running > 0) {
+      estimatedTimeRemainingMs = Math.round(avgExecutionTimeMs * (pending + running))
+    }
+
+    return {
+      total,
+      pending,
+      running,
+      completed,
+      failed,
+      skipped,
+      percentComplete,
+      estimatedTimeRemainingMs,
+      avgExecutionTimeMs: Math.round(avgExecutionTimeMs),
+    }
+  }
+
+  /**
+   * Start tracking a migration operation
+   *
+   * @param migrationId Unique identifier for this migration operation
+   * @param doIds Array of Durable Object IDs to migrate
+   * @param targetVersion Target schema version
+   * @param options Optional operation metadata
+   * @returns The created migration operation
+   */
+  async startMigration(
+    migrationId: string,
+    doIds: string[],
+    targetVersion: number,
+    options?: {
+      initiatedBy?: string
+      description?: string
+    }
+  ): Promise<MigrationOperation> {
+    this.log(`Starting migration ${migrationId} for ${doIds.length} DOs to version ${targetVersion}`)
+
+    // Check if migration already exists
+    const existing = await this.storage.getOperation(migrationId)
+    if (existing) {
+      throw new Error(`Migration operation ${migrationId} already exists`)
+    }
+
+    // Create operation record
+    const operation: MigrationOperation = {
+      migrationId,
+      targetVersion,
+      status: 'pending',
+      totalDOs: doIds.length,
+      startedAt: new Date().toISOString(),
+      ...(options?.initiatedBy !== undefined && { initiatedBy: options.initiatedBy }),
+      ...(options?.description !== undefined && { description: options.description }),
+      stats: {
+        total: doIds.length,
+        pending: doIds.length,
+        running: 0,
+        completed: 0,
+        failed: 0,
+        skipped: 0,
+        percentComplete: 0,
+        estimatedTimeRemainingMs: null,
+        avgExecutionTimeMs: 0,
+      },
+    }
+
+    await this.storage.setOperation(migrationId, operation)
+
+    // Initialize all DOs as pending
+    for (const doId of doIds) {
+      const doProgress: DOProgressState = {
+        doId,
+        status: 'pending',
+        currentVersion: null,
+        targetVersion,
+        migrationsCompleted: 0,
+        totalMigrations: 0, // Will be updated when migration starts
+      }
+      await this.storage.setDOProgress(migrationId, doId, doProgress)
+    }
+
+    this.log(`Migration ${migrationId} started`)
+    return operation
+  }
+
+  /**
+   * Report progress for a specific DO
+   *
+   * @param migrationId Migration operation ID
+   * @param progress Progress update for a DO
+   */
+  async reportProgress(migrationId: string, progress: Partial<DOProgressState> & { doId: string }): Promise<void> {
+    const operation = await this.storage.getOperation(migrationId)
+    if (!operation) {
+      throw new Error(`Migration operation ${migrationId} not found`)
+    }
+
+    // Get existing progress or create new
+    const existing = await this.storage.getDOProgress(migrationId, progress.doId)
+    if (!existing) {
+      throw new Error(`DO ${progress.doId} not registered for migration ${migrationId}`)
+    }
+
+    // Merge updates
+    const updated: DOProgressState = {
+      ...existing,
+      ...progress,
+    }
+
+    // Set timestamps
+    if (progress.status === 'running' && !existing.startedAt) {
+      updated.startedAt = new Date().toISOString()
+    }
+    if (
+      (progress.status === 'completed' || progress.status === 'failed' || progress.status === 'skipped') &&
+      !existing.completedAt
+    ) {
+      updated.completedAt = new Date().toISOString()
+      // Calculate execution time if we have start time
+      if (updated.startedAt) {
+        updated.executionTimeMs =
+          new Date(updated.completedAt).getTime() - new Date(updated.startedAt).getTime()
+      }
+    }
+
+    await this.storage.setDOProgress(migrationId, progress.doId, updated)
+
+    // Update operation stats and status
+    const stats = await this.calculateStats(migrationId)
+    operation.stats = stats
+
+    // Update operation status based on progress
+    if (operation.status === 'pending' && stats.running > 0) {
+      operation.status = 'running'
+    } else if (stats.running === 0 && stats.pending === 0) {
+      if (stats.failed > 0) {
+        operation.status = 'failed'
+      } else {
+        operation.status = 'completed'
+      }
+      operation.completedAt = new Date().toISOString()
+    }
+
+    await this.storage.setOperation(migrationId, operation)
+
+    this.log(
+      `Progress for ${progress.doId}: ${progress.status} (${stats.percentComplete}% overall)`
+    )
+  }
+
+  /**
+   * Get overall progress for a migration operation
+   *
+   * @param migrationId Migration operation ID
+   * @returns Migration operation with current stats
+   */
+  async getProgress(migrationId: string): Promise<MigrationOperation> {
+    const operation = await this.storage.getOperation(migrationId)
+    if (!operation) {
+      throw new Error(`Migration operation ${migrationId} not found`)
+    }
+
+    // Refresh stats
+    const stats = await this.calculateStats(migrationId)
+    operation.stats = stats
+
+    return operation
+  }
+
+  /**
+   * Get DOs that are still pending migration
+   *
+   * @param migrationId Migration operation ID
+   * @returns Array of DO IDs that haven't started migration
+   */
+  async getPendingDOs(migrationId: string): Promise<string[]> {
+    const pending = await this.storage.getDOProgressByStatus(migrationId, 'pending')
+    return pending.map((p) => p.doId)
+  }
+
+  /**
+   * Get DOs that are currently running migration
+   *
+   * @param migrationId Migration operation ID
+   * @returns Array of DO IDs currently migrating
+   */
+  async getRunningDOs(migrationId: string): Promise<string[]> {
+    const running = await this.storage.getDOProgressByStatus(migrationId, 'running')
+    return running.map((p) => p.doId)
+  }
+
+  /**
+   * Get DOs that failed migration
+   *
+   * @param migrationId Migration operation ID
+   * @returns Array of progress states for failed DOs
+   */
+  async getFailedDOs(migrationId: string): Promise<DOProgressState[]> {
+    return this.storage.getDOProgressByStatus(migrationId, 'failed')
+  }
+
+  /**
+   * Get progress for a specific DO
+   *
+   * @param migrationId Migration operation ID
+   * @param doId Durable Object ID
+   * @returns Progress state for the DO
+   */
+  async getDOProgress(migrationId: string, doId: string): Promise<DOProgressState | null> {
+    return this.storage.getDOProgress(migrationId, doId)
+  }
+
+  /**
+   * Pause a migration operation
+   *
+   * This doesn't stop in-flight migrations but marks the operation as paused,
+   * preventing new migrations from starting.
+   *
+   * @param migrationId Migration operation ID
+   */
+  async pauseMigration(migrationId: string): Promise<void> {
+    const operation = await this.storage.getOperation(migrationId)
+    if (!operation) {
+      throw new Error(`Migration operation ${migrationId} not found`)
+    }
+
+    if (operation.status !== 'running') {
+      throw new Error(`Cannot pause migration ${migrationId} with status ${operation.status}`)
+    }
+
+    operation.status = 'paused'
+    await this.storage.setOperation(migrationId, operation)
+
+    this.log(`Migration ${migrationId} paused`)
+  }
+
+  /**
+   * Resume a paused migration operation
+   *
+   * @param migrationId Migration operation ID
+   */
+  async resumeMigration(migrationId: string): Promise<void> {
+    const operation = await this.storage.getOperation(migrationId)
+    if (!operation) {
+      throw new Error(`Migration operation ${migrationId} not found`)
+    }
+
+    if (operation.status !== 'paused') {
+      throw new Error(`Cannot resume migration ${migrationId} with status ${operation.status}`)
+    }
+
+    operation.status = 'running'
+    await this.storage.setOperation(migrationId, operation)
+
+    this.log(`Migration ${migrationId} resumed`)
+  }
+
+  /**
+   * Cancel a migration operation
+   *
+   * This marks the operation as failed and stops tracking.
+   * It doesn't rollback any completed migrations.
+   *
+   * @param migrationId Migration operation ID
+   */
+  async cancelMigration(migrationId: string): Promise<void> {
+    const operation = await this.storage.getOperation(migrationId)
+    if (!operation) {
+      throw new Error(`Migration operation ${migrationId} not found`)
+    }
+
+    if (operation.status === 'completed' || operation.status === 'failed') {
+      throw new Error(`Cannot cancel migration ${migrationId} with status ${operation.status}`)
+    }
+
+    operation.status = 'failed'
+    operation.completedAt = new Date().toISOString()
+    await this.storage.setOperation(migrationId, operation)
+
+    this.log(`Migration ${migrationId} cancelled`)
+  }
+
+  /**
+   * Delete a migration operation and all progress data
+   *
+   * @param migrationId Migration operation ID
+   */
+  async deleteMigration(migrationId: string): Promise<void> {
+    await this.storage.deleteOperation(migrationId)
+    this.log(`Migration ${migrationId} deleted`)
+  }
+
+  /**
+   * List all migration operations
+   *
+   * @returns Array of all migration operations
+   */
+  async listMigrations(): Promise<MigrationOperation[]> {
+    return this.storage.listOperations()
+  }
+
+  /**
+   * Retry failed DOs in a migration operation
+   *
+   * Resets failed DOs to pending status so they can be retried.
+   *
+   * @param migrationId Migration operation ID
+   * @returns Number of DOs reset to pending
+   */
+  async retryFailedDOs(migrationId: string): Promise<number> {
+    const failed = await this.getFailedDOs(migrationId)
+
+    for (const progress of failed) {
+      // Create new object without error/completedAt/executionTimeMs
+      const { error: _error, completedAt: _completedAt, executionTimeMs: _execTime, ...rest } = progress
+      await this.storage.setDOProgress(migrationId, progress.doId, {
+        ...rest,
+        status: 'pending',
+      })
+    }
+
+    // Update operation stats
+    const operation = await this.storage.getOperation(migrationId)
+    if (operation) {
+      const stats = await this.calculateStats(migrationId)
+      operation.stats = stats
+      if (operation.status === 'failed' || operation.status === 'completed') {
+        operation.status = 'running'
+        delete operation.completedAt
+      }
+      await this.storage.setOperation(migrationId, operation)
+    }
+
+    this.log(`Reset ${failed.length} failed DOs to pending in migration ${migrationId}`)
+    return failed.length
+  }
+}
+
+/**
+ * Create a migration progress tracker
+ *
+ * @param config Progress tracker configuration
+ * @returns Configured progress tracker instance
+ *
+ * @example
+ * ```typescript
+ * const tracker = createProgressTracker({
+ *   storage: new InMemoryProgressStorage(),
+ *   debug: true,
+ * })
+ * ```
+ */
+export function createProgressTracker(config: ProgressTrackerConfig): MigrationProgressTracker {
+  return new MigrationProgressTracker(config)
+}