@dotdo/postgres 0.1.0

package/src/migrations/bulk-orchestrator.ts

@@ -0,0 +1,1039 @@
+/**
+ * Bulk Migration Orchestrator
+ *
+ * Orchestrates large-scale migrations across many Durable Object instances.
+ * When migrating existing user DOs to a new schema version, this provides
+ * batch processing, concurrency control, retry logic, and rollback coordination.
+ *
+ * Key features:
+ * - Queue-based migration dispatch using Cloudflare Queues
+ * - Batch processing with configurable concurrency limits
+ * - Automatic retry logic for failed migrations
+ * - Coordinated rollback across multiple DOs
+ * - Progress tracking and status monitoring
+ *
+ * @module migrations/bulk-orchestrator
+ */
+
+import type { Migration, QueryExecutor } from './types'
+import type { MigrationRegistry } from './registry'
+import { createLogger, LogLevel, type ILogger } from '@dotdo/postgres-shared'
+
+// Re-export batch types for external use
+export type { BatchMigrationResult, BatchRollbackResult } from './types'
+
+// Module-level logger for BulkMigrationOrchestrator
+// Level is DEBUG because the debug config flag gates whether log() is called at all.
+// The logger just provides structured output - the filtering happens at call site.
+const bulkOrchestratorLogger: ILogger = createLogger({
+  level: LogLevel.DEBUG,
+  prefix: '[BulkMigrationOrchestrator]',
+})
+
+/**
+ * Selector for targeting DOs to migrate
+ */
+export interface DOSelector {
+  /**
+   * Filter by DO ID prefix (e.g., 'user_')
+   */
+  idPrefix?: string
+
+  /**
+   * Filter by DO IDs (explicit list)
+   */
+  ids?: string[]
+
+  /**
+   * Maximum number of DOs to select
+   */
+  limit?: number
+
+  /**
+   * Custom filter function
+   */
+  filter?: (id: string) => boolean
+}
+
+/**
+ * Migration job status
+ */
+export type MigrationJobStatus =
+  | 'pending'
+  | 'queued'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'retrying'
+  | 'rolled_back'
+
+/**
+ * Individual DO migration job
+ */
+export interface DOBulkMigrationJob {
+  /** Unique job ID */
+  jobId: string
+
+  /** Migration ID being applied */
+  migrationId: string
+
+  /** DO ID to migrate */
+  doId: string
+
+  /** Current job status */
+  status: MigrationJobStatus
+
+  /** Number of retry attempts */
+  retryCount: number
+
+  /** Maximum retry attempts */
+  maxRetries: number
+
+  /** Error message if failed */
+  error?: string
+
+  /** When the job was created */
+  createdAt: Date
+
+  /** When the job was last updated */
+  updatedAt: Date
+
+  /** When the job completed (success or failure) */
+  completedAt?: Date
+
+  /** Execution time in milliseconds */
+  executionTimeMs?: number
+}
+
+/**
+ * Bulk migration batch
+ */
+export interface MigrationBatch {
+  /** Batch ID */
+  batchId: string
+
+  /** Migration ID */
+  migrationId: string
+
+  /** Jobs in this batch */
+  jobs: DOBulkMigrationJob[]
+
+  /** Total jobs in batch */
+  totalJobs: number
+
+  /** Completed jobs */
+  completedJobs: number
+
+  /** Failed jobs */
+  failedJobs: number
+
+  /** When the batch started */
+  startedAt: Date
+
+  /** When the batch completed */
+  completedAt?: Date
+}
+
+/**
+ * Bulk migration progress
+ */
+export interface BulkMigrationProgress {
+  /** Migration ID */
+  migrationId: string
+
+  /** Total DOs to migrate */
+  total: number
+
+  /** Successfully migrated */
+  completed: number
+
+  /** Failed migrations */
+  failed: number
+
+  /** Currently running */
+  running: number
+
+  /** Queued for processing */
+  queued: number
+
+  /** Overall progress percentage (0-100) */
+  progressPercent: number
+}
+
+/**
+ * Cloudflare Queue message for migration dispatch
+ */
+export interface MigrationQueueMessage {
+  /** Message type */
+  type: 'migrate' | 'rollback' | 'retry'
+
+  /** Migration ID */
+  migrationId: string
+
+  /** DO ID to process */
+  doId: string
+
+  /** Job ID for tracking */
+  jobId: string
+
+  /** Retry attempt number */
+  retryAttempt: number
+
+  /** Batch ID for coordination */
+  batchId: string
+}
+
+/**
+ * Bulk migration orchestrator configuration
+ */
+export interface BulkOrchestratorConfig {
+  /**
+   * Migration registry
+   */
+  registry: MigrationRegistry
+
+  /**
+   * Cloudflare Queue binding for dispatching migrations
+   */
+  queue?: Queue<MigrationQueueMessage>
+
+  /**
+   * Maximum number of concurrent migrations
+   * Default: 10
+   */
+  concurrency?: number
+
+  /**
+   * Maximum retry attempts for failed migrations
+   * Default: 3
+   */
+  maxRetries?: number
+
+  /**
+   * Retry delay in milliseconds
+   * Default: 1000 (1 second)
+   */
+  retryDelayMs?: number
+
+  /**
+   * Batch size for processing
+   * Default: 100
+   */
+  batchSize?: number
+
+  /**
+   * Storage for job state (KV, DO, etc.)
+   */
+  storage?: BulkMigrationStorage
+
+  /**
+   * Enable debug logging
+   * Default: false
+   */
+  debug?: boolean
+}
+
+/**
+ * Storage interface for bulk migration state
+ */
+export interface BulkMigrationStorage {
+  /**
+   * Get a migration job by ID
+   */
+  getJob(jobId: string): Promise<DOBulkMigrationJob | null>
+
+  /**
+   * Save a migration job
+   */
+  saveJob(job: DOBulkMigrationJob): Promise<void>
+
+  /**
+   * Get all jobs for a migration
+   */
+  getJobsForMigration(migrationId: string): Promise<DOBulkMigrationJob[]>
+
+  /**
+   * Get batch by ID
+   */
+  getBatch(batchId: string): Promise<MigrationBatch | null>
+
+  /**
+   * Save batch
+   */
+  saveBatch(batch: MigrationBatch): Promise<void>
+
+  /**
+   * Delete job
+   */
+  deleteJob(jobId: string): Promise<void>
+}
+
+/**
+ * In-memory storage implementation (for testing/development)
+ */
+export class InMemoryBulkMigrationStorage implements BulkMigrationStorage {
+  private jobs: Map<string, DOBulkMigrationJob> = new Map()
+  private batches: Map<string, MigrationBatch> = new Map()
+
+  async getJob(jobId: string): Promise<DOBulkMigrationJob | null> {
+    return this.jobs.get(jobId) ?? null
+  }
+
+  async saveJob(job: DOBulkMigrationJob): Promise<void> {
+    this.jobs.set(job.jobId, job)
+  }
+
+  async getJobsForMigration(migrationId: string): Promise<DOBulkMigrationJob[]> {
+    return Array.from(this.jobs.values()).filter((job) => job.migrationId === migrationId)
+  }
+
+  async getBatch(batchId: string): Promise<MigrationBatch | null> {
+    return this.batches.get(batchId) ?? null
+  }
+
+  async saveBatch(batch: MigrationBatch): Promise<void> {
+    this.batches.set(batch.batchId, batch)
+  }
+
+  async deleteJob(jobId: string): Promise<void> {
+    this.jobs.delete(jobId)
+  }
+
+  /**
+   * Clear all data (for testing)
+   */
+  clear(): void {
+    this.jobs.clear()
+    this.batches.clear()
+  }
+}
+
+/**
+ * Generate a unique ID
+ */
+function generateId(prefix: string): string {
+  const timestamp = Date.now().toString(36)
+  const random = Math.random().toString(36).substring(2, 10)
+  return `${prefix}_${timestamp}_${random}`
+}
+
+/**
+ * Bulk Migration Orchestrator
+ *
+ * Coordinates migrations across many Durable Object instances.
+ *
+ * @example
+ * ```typescript
+ * const orchestrator = new BulkMigrationOrchestrator({
+ *   registry: migrationRegistry,
+ *   queue: env.MIGRATION_QUEUE,
+ *   concurrency: 20,
+ *   maxRetries: 3,
+ * })
+ *
+ * // Schedule migration for all user DOs
+ * const batch = await orchestrator.scheduleMigration('0005_add_field', {
+ *   idPrefix: 'user_',
+ * })
+ *
+ * // Check progress
+ * const progress = await orchestrator.getProgress('0005_add_field')
+ * console.log(`Progress: ${progress.progressPercent}%`)
+ *
+ * // Retry failed migrations
+ * await orchestrator.retryFailed('0005_add_field')
+ *
+ * // Rollback if needed
+ * await orchestrator.rollbackAll('0005_add_field')
+ * ```
+ */
+export class BulkMigrationOrchestrator {
+  private registry: MigrationRegistry
+  private queue: Queue<MigrationQueueMessage> | undefined
+  private config: Required<Omit<BulkOrchestratorConfig, 'registry' | 'queue' | 'storage'>>
+  private storage: BulkMigrationStorage
+  /** @internal Reserved for future concurrent batch tracking */
+  private _activeBatches: Map<string, Promise<void>> = new Map()
+
+  constructor(config: BulkOrchestratorConfig) {
+    void this._activeBatches // Reserved for future concurrent batch tracking
+    this.registry = config.registry
+    this.queue = config.queue
+    this.config = {
+      concurrency: config.concurrency ?? 10,
+      maxRetries: config.maxRetries ?? 3,
+      retryDelayMs: config.retryDelayMs ?? 1000,
+      batchSize: config.batchSize ?? 100,
+      debug: config.debug ?? false,
+    }
+    this.storage = config.storage ?? new InMemoryBulkMigrationStorage()
+  }
+
+  /**
+   * Log debug message
+   */
+  private log(message: string, ...args: unknown[]): void {
+    if (this.config.debug) {
+      bulkOrchestratorLogger.debug(message, args.length > 0 ? { args } : undefined)
+    }
+  }
+
+  /**
+   * Get DO IDs matching the selector
+   *
+   * Note: In production, this would typically query a DO namespace listing
+   * or a registry of active DOs. This is a simplified implementation.
+   */
+  private async getDOIds(
+    selector: DOSelector,
+    getDOIdsFn?: (selector: DOSelector) => Promise<string[]>
+  ): Promise<string[]> {
+    // If a custom function is provided, use it
+    if (getDOIdsFn) {
+      return getDOIdsFn(selector)
+    }
+
+    // Otherwise, require explicit IDs
+    if (!selector.ids || selector.ids.length === 0) {
+      throw new Error(
+        'selector.ids must be provided, or pass getDOIdsFn to scheduleMigration()'
+      )
+    }
+
+    let ids = selector.ids
+
+    // Apply prefix filter
+    if (selector.idPrefix) {
+      ids = ids.filter((id) => id.startsWith(selector.idPrefix!))
+    }
+
+    // Apply custom filter
+    if (selector.filter) {
+      ids = ids.filter(selector.filter)
+    }
+
+    // Apply limit
+    if (selector.limit) {
+      ids = ids.slice(0, selector.limit)
+    }
+
+    return ids
+  }
+
+  /**
+   * Schedule a migration for all DOs matching the selector
+   *
+   * @param migrationId Migration ID to run
+   * @param selector DO selector
+   * @param getDOIdsFn Optional function to get DO IDs (for production use)
+   * @returns Batch information
+   *
+   * @example
+   * ```typescript
+   * // With explicit IDs
+   * await orchestrator.scheduleMigration('0005_add_field', {
+   *   ids: ['user_1', 'user_2', 'user_3'],
+   * })
+   *
+   * // With custom DO listing function
+   * await orchestrator.scheduleMigration('0005_add_field', {
+   *   idPrefix: 'user_',
+   * }, async (selector) => {
+   *   // Query your DO namespace or registry
+   *   return await listActiveDOs(selector)
+   * })
+   * ```
+   */
+  async scheduleMigration(
+    migrationId: string,
+    selector: DOSelector,
+    getDOIdsFn?: (selector: DOSelector) => Promise<string[]>
+  ): Promise<MigrationBatch> {
+    this.log(`Scheduling migration ${migrationId}`)
+
+    // Validate migration exists
+    const migration = this.registry.getMigration(migrationId)
+    if (!migration) {
+      throw new Error(`Migration ${migrationId} not found in registry`)
+    }
+
+    // Get DO IDs to migrate
+    const doIds = await this.getDOIds(selector, getDOIdsFn)
+    this.log(`Found ${doIds.length} DOs to migrate`)
+
+    // Create batch
+    const batchId = generateId('batch')
+    const batch: MigrationBatch = {
+      batchId,
+      migrationId,
+      jobs: [],
+      totalJobs: doIds.length,
+      completedJobs: 0,
+      failedJobs: 0,
+      startedAt: new Date(),
+    }
+
+    // Create jobs for each DO
+    const jobs: DOBulkMigrationJob[] = []
+    for (const doId of doIds) {
+      const jobId = generateId('job')
+      const job: DOBulkMigrationJob = {
+        jobId,
+        migrationId,
+        doId,
+        status: 'pending',
+        retryCount: 0,
+        maxRetries: this.config.maxRetries,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      }
+      jobs.push(job)
+      await this.storage.saveJob(job)
+    }
+
+    batch.jobs = jobs
+    await this.storage.saveBatch(batch)
+
+    // Queue the jobs
+    if (this.queue) {
+      await this.queueJobs(batch, jobs)
+    }
+
+    this.log(`Batch ${batchId} created with ${jobs.length} jobs`)
+
+    return batch
+  }
+
+  /**
+   * Queue jobs for processing
+   */
+  private async queueJobs(batch: MigrationBatch, jobs: DOBulkMigrationJob[]): Promise<void> {
+    if (!this.queue) {
+      throw new Error('Queue not configured')
+    }
+
+    // Send jobs to queue in batches
+    for (let i = 0; i < jobs.length; i += this.config.batchSize) {
+      const batchJobs = jobs.slice(i, i + this.config.batchSize)
+      const messages: MessageSendRequest<MigrationQueueMessage>[] = batchJobs.map((job) => ({
+        body: {
+          type: 'migrate',
+          migrationId: job.migrationId,
+          doId: job.doId,
+          jobId: job.jobId,
+          retryAttempt: 0,
+          batchId: batch.batchId,
+        },
+      }))
+
+      await this.queue.sendBatch(messages)
+
+      // Update job status to queued
+      for (const job of batchJobs) {
+        job.status = 'queued'
+        job.updatedAt = new Date()
+        await this.storage.saveJob(job)
+      }
+    }
+  }
+
+  /**
+   * Process a batch of migration jobs with concurrency control
+   *
+   * @param batch Jobs to process
+   * @param getExecutor Function to get QueryExecutor for a DO ID
+   * @returns Processing results
+   *
+   * @example
+   * ```typescript
+   * const batch = await orchestrator.getBatch(batchId)
+   * await orchestrator.processBatch(batch, async (doId) => {
+   *   const stub = env.MY_DO.get(env.MY_DO.idFromName(doId))
+   *   return stub.getExecutor()
+   * })
+   * ```
+   */
+  async processBatch(
+    batch: MigrationBatch,
+    getExecutor: (doId: string) => Promise<QueryExecutor>
+  ): Promise<{
+    completed: number
+    failed: number
+    results: Array<{ jobId: string; success: boolean; error?: string }>
+  }> {
+    this.log(`Processing batch ${batch.batchId} with ${batch.jobs.length} jobs`)
+
+    const results: Array<{ jobId: string; success: boolean; error?: string }> = []
+    let completed = 0
+    let failed = 0
+
+    // Process jobs with concurrency control
+    const activeJobs: Promise<void>[] = []
+
+    for (const job of batch.jobs) {
+      // Skip already completed jobs
+      if (job.status === 'completed' || job.status === 'rolled_back') {
+        continue
+      }
+
+      // Wait if we've hit concurrency limit
+      if (activeJobs.length >= this.config.concurrency) {
+        await Promise.race(activeJobs)
+      }
+
+      // Process job
+      const jobPromise = this.processJob(job, getExecutor)
+        .then((result) => {
+          if (result.success) {
+            completed++
+          } else {
+            failed++
+          }
+          results.push(result)
+
+          // Remove from active jobs
+          const index = activeJobs.indexOf(jobPromise)
+          if (index >= 0) {
+            activeJobs.splice(index, 1)
+          }
+        })
+        .catch((error) => {
+          this.log(`Job ${job.jobId} error:`, error)
+          failed++
+          results.push({
+            jobId: job.jobId,
+            success: false,
+            error: error instanceof Error ? error.message : String(error),
+          })
+
+          // Remove from active jobs
+          const index = activeJobs.indexOf(jobPromise)
+          if (index >= 0) {
+            activeJobs.splice(index, 1)
+          }
+        })
+
+      activeJobs.push(jobPromise)
+    }
+
+    // Wait for all remaining jobs
+    await Promise.all(activeJobs)
+
+    // Update batch
+    batch.completedJobs = completed
+    batch.failedJobs = failed
+    if (completed + failed >= batch.totalJobs) {
+      batch.completedAt = new Date()
+    }
+    await this.storage.saveBatch(batch)
+
+    this.log(`Batch ${batch.batchId} processing complete: ${completed} succeeded, ${failed} failed`)
+
+    return { completed, failed, results }
+  }
+
+  /**
+   * Process a single migration job
+   */
+  private async processJob(
+    job: DOBulkMigrationJob,
+    getExecutor: (doId: string) => Promise<QueryExecutor>
+  ): Promise<{ jobId: string; success: boolean; error?: string }> {
+    this.log(`Processing job ${job.jobId} for DO ${job.doId}`)
+
+    const startTime = performance.now()
+
+    try {
+      // Update job status
+      job.status = 'running'
+      job.updatedAt = new Date()
+      await this.storage.saveJob(job)
+
+      // Get executor for this DO
+      const executor = await getExecutor(job.doId)
+
+      // Get migration
+      const migration = this.registry.getMigration(job.migrationId)
+      if (!migration) {
+        throw new Error(`Migration ${job.migrationId} not found`)
+      }
+
+      // Execute migration
+      await this.executeMigration(executor, migration)
+
+      // Update job as completed
+      const executionTimeMs = Math.round(performance.now() - startTime)
+      job.status = 'completed'
+      job.completedAt = new Date()
+      job.updatedAt = new Date()
+      job.executionTimeMs = executionTimeMs
+      await this.storage.saveJob(job)
+
+      this.log(`Job ${job.jobId} completed in ${executionTimeMs}ms`)
+
+      return { jobId: job.jobId, success: true }
+    } catch (error) {
+      const executionTimeMs = Math.round(performance.now() - startTime)
+      const errorMessage = error instanceof Error ? error.message : String(error)
+
+      this.log(`Job ${job.jobId} failed:`, errorMessage)
+
+      // Check if we should retry
+      if (job.retryCount < job.maxRetries) {
+        job.status = 'retrying'
+        job.retryCount++
+        job.error = errorMessage
+        job.updatedAt = new Date()
+        job.executionTimeMs = executionTimeMs
+        await this.storage.saveJob(job)
+
+        // Schedule retry (with exponential backoff)
+        const retryDelay = this.config.retryDelayMs * Math.pow(2, job.retryCount - 1)
+        await new Promise((resolve) => setTimeout(resolve, retryDelay))
+
+        // Retry
+        return this.processJob(job, getExecutor)
+      } else {
+        // Max retries exceeded
+        job.status = 'failed'
+        job.error = errorMessage
+        job.completedAt = new Date()
+        job.updatedAt = new Date()
+        job.executionTimeMs = executionTimeMs
+        await this.storage.saveJob(job)
+
+        return { jobId: job.jobId, success: false, error: errorMessage }
+      }
+    }
+  }
+
+  /**
+   * Execute a migration on a single DO
+   */
+  private async executeMigration(
+    executor: QueryExecutor,
+    migration: Migration
+  ): Promise<void> {
+    // Check if migration is already applied
+    const checkResult = await executor.query<{ count: number }>(
+      `SELECT COUNT(*) as count FROM _migrations WHERE id = $1 AND status = 'applied'`,
+      [migration.id]
+    )
+
+    if (checkResult.rows[0]?.count && checkResult.rows[0].count > 0) {
+      this.log(`Migration ${migration.id} already applied, skipping`)
+      return
+    }
+
+    // Execute migration in a transaction
+    await executor.query('BEGIN')
+
+    try {
+      // Run migration SQL
+      await executor.query(migration.up)
+
+      // Record migration
+      const checksum = await this.calculateChecksum(migration.up)
+      await executor.query(
+        `INSERT INTO _migrations (id, name, version, checksum, execution_time_ms, status)
+         VALUES ($1, $2, $3, $4, 0, 'applied')
+         ON CONFLICT (id) DO UPDATE SET
+           status = 'applied',
+           applied_at = CURRENT_TIMESTAMP`,
+        [migration.id, migration.name, migration.version, checksum]
+      )
+
+      await executor.query('COMMIT')
+    } catch (error) {
+      await executor.query('ROLLBACK')
+      throw error
+    }
+  }
+
+  /**
+   * Calculate checksum for migration content
+   */
+  private async calculateChecksum(content: string): Promise<string> {
+    const encoder = new TextEncoder()
+    const data = encoder.encode(content)
+    const hashBuffer = await crypto.subtle.digest('SHA-256', data)
+    const hashArray = Array.from(new Uint8Array(hashBuffer))
+    return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('')
+  }
+
+  /**
+   * Get migration progress
+   *
+   * @param migrationId Migration ID to check
+   * @returns Progress information
+   */
+  async getProgress(migrationId: string): Promise<BulkMigrationProgress> {
+    const jobs = await this.storage.getJobsForMigration(migrationId)
+
+    const total = jobs.length
+    const completed = jobs.filter((j) => j.status === 'completed').length
+    const failed = jobs.filter((j) => j.status === 'failed').length
+    const running = jobs.filter((j) => j.status === 'running').length
+    const queued = jobs.filter((j) => j.status === 'queued' || j.status === 'pending').length
+
+    const progressPercent = total > 0 ? Math.round((completed / total) * 100) : 0
+
+    return {
+      migrationId,
+      total,
+      completed,
+      failed,
+      running,
+      queued,
+      progressPercent,
+    }
+  }
+
+  /**
+   * Retry all failed migrations for a migration ID
+   *
+   * @param migrationId Migration ID
+   * @param getExecutor Function to get QueryExecutor for a DO ID
+   * @returns Number of jobs retried
+   */
+  async retryFailed(
+    migrationId: string,
+    getExecutor: (doId: string) => Promise<QueryExecutor>
+  ): Promise<number> {
+    this.log(`Retrying failed migrations for ${migrationId}`)
+
+    const jobs = await this.storage.getJobsForMigration(migrationId)
+    const failedJobs = jobs.filter((j) => j.status === 'failed')
+
+    this.log(`Found ${failedJobs.length} failed jobs to retry`)
+
+    // Reset retry count and status
+    for (const job of failedJobs) {
+      job.retryCount = 0
+      job.status = 'pending'
+      delete job.error
+      job.updatedAt = new Date()
+      await this.storage.saveJob(job)
+    }
+
+    // Create a temporary batch for processing
+    const batchId = generateId('retry_batch')
+    const batch: MigrationBatch = {
+      batchId,
+      migrationId,
+      jobs: failedJobs,
+      totalJobs: failedJobs.length,
+      completedJobs: 0,
+      failedJobs: 0,
+      startedAt: new Date(),
+    }
+
+    // Process the batch
+    await this.processBatch(batch, getExecutor)
+
+    return failedJobs.length
+  }
+
+  /**
+   * Rollback all DOs for a migration
+   *
+   * Coordinates rollback across all DOs that have this migration applied.
+   *
+   * @param migrationId Migration ID to rollback
+   * @param getExecutor Function to get QueryExecutor for a DO ID
+   * @returns Rollback results
+   */
+  async rollbackAll(
+    migrationId: string,
+    getExecutor: (doId: string) => Promise<QueryExecutor>
+  ): Promise<{
+    total: number
+    succeeded: number
+    failed: number
+    results: Array<{ doId: string; success: boolean; error?: string }>
+  }> {
+    this.log(`Rolling back migration ${migrationId}`)
+
+    // Get migration
+    const migration = this.registry.getMigration(migrationId)
+    if (!migration) {
+      throw new Error(`Migration ${migrationId} not found in registry`)
+    }
+
+    if (!migration.down) {
+      throw new Error(`Migration ${migrationId} has no rollback SQL defined`)
+    }
+
+    // Get all jobs for this migration that were completed
+    const jobs = await this.storage.getJobsForMigration(migrationId)
+    const completedJobs = jobs.filter((j) => j.status === 'completed')
+
+    this.log(`Found ${completedJobs.length} completed migrations to rollback`)
+
+    const results: Array<{ doId: string; success: boolean; error?: string }> = []
+    let succeeded = 0
+    let failed = 0
+
+    // Process rollbacks with concurrency control
+    const activeRollbacks: Promise<void>[] = []
+
+    for (const job of completedJobs) {
+      // Wait if we've hit concurrency limit
+      if (activeRollbacks.length >= this.config.concurrency) {
+        await Promise.race(activeRollbacks)
+      }
+
+      const rollbackPromise = this.rollbackJob(job, migration, getExecutor)
+        .then((result) => {
+          if (result.success) {
+            succeeded++
+          } else {
+            failed++
+          }
+          results.push(result)
+
+          // Remove from active rollbacks
+          const index = activeRollbacks.indexOf(rollbackPromise)
+          if (index >= 0) {
+            activeRollbacks.splice(index, 1)
+          }
+        })
+        .catch((error) => {
+          this.log(`Rollback for DO ${job.doId} error:`, error)
+          failed++
+          results.push({
+            doId: job.doId,
+            success: false,
+            error: error instanceof Error ? error.message : String(error),
+          })
+
+          // Remove from active rollbacks
+          const index = activeRollbacks.indexOf(rollbackPromise)
+          if (index >= 0) {
+            activeRollbacks.splice(index, 1)
+          }
+        })
+
+      activeRollbacks.push(rollbackPromise)
+    }
+
+    // Wait for all rollbacks
+    await Promise.all(activeRollbacks)
+
+    this.log(
+      `Rollback complete for ${migrationId}: ${succeeded} succeeded, ${failed} failed`
+    )
+
+    return {
+      total: completedJobs.length,
+      succeeded,
+      failed,
+      results,
+    }
+  }
+
+  /**
+   * Rollback a single migration job
+   */
+  private async rollbackJob(
+    job: DOBulkMigrationJob,
+    migration: Migration,
+    getExecutor: (doId: string) => Promise<QueryExecutor>
+  ): Promise<{ doId: string; success: boolean; error?: string }> {
+    this.log(`Rolling back job ${job.jobId} for DO ${job.doId}`)
+
+    try {
+      const executor = await getExecutor(job.doId)
+
+      // Execute rollback in a transaction
+      await executor.query('BEGIN')
+
+      try {
+        await executor.query(migration.down!)
+        await executor.query(
+          `UPDATE _migrations SET status = 'rolled_back', applied_at = CURRENT_TIMESTAMP WHERE id = $1`,
+          [migration.id]
+        )
+        await executor.query('COMMIT')
+
+        // Update job status
+        job.status = 'rolled_back'
+        job.updatedAt = new Date()
+        await this.storage.saveJob(job)
+
+        return { doId: job.doId, success: true }
+      } catch (error) {
+        await executor.query('ROLLBACK')
+        throw error
+      }
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error)
+      this.log(`Rollback failed for DO ${job.doId}:`, errorMessage)
+      return { doId: job.doId, success: false, error: errorMessage }
+    }
+  }
+
+  /**
+   * Get batch by ID
+   */
+  async getBatch(batchId: string): Promise<MigrationBatch | null> {
+    return this.storage.getBatch(batchId)
+  }
+
+  /**
+   * Get job by ID
+   */
+  async getJob(jobId: string): Promise<DOBulkMigrationJob | null> {
+    return this.storage.getJob(jobId)
+  }
+
+  /**
+   * Get all jobs for a migration
+   */
+  async getJobs(migrationId: string): Promise<DOBulkMigrationJob[]> {
+    return this.storage.getJobsForMigration(migrationId)
+  }
+
+  /**
+   * Cancel a batch
+   *
+   * Cancels all pending/queued jobs in a batch.
+   */
+  async cancelBatch(batchId: string): Promise<number> {
+    const batch = await this.storage.getBatch(batchId)
+    if (!batch) {
+      throw new Error(`Batch ${batchId} not found`)
+    }
+
+    let cancelled = 0
+    for (const job of batch.jobs) {
+      if (job.status === 'pending' || job.status === 'queued') {
+        await this.storage.deleteJob(job.jobId)
+        cancelled++
+      }
+    }
+
+    this.log(`Cancelled ${cancelled} jobs in batch ${batchId}`)
+
+    return cancelled
+  }
+}
+
+/**
+ * Create a bulk migration orchestrator
+ *
+ * @example
+ * ```typescript
+ * const orchestrator = createBulkMigrationOrchestrator({
+ *   registry: migrationRegistry,
+ *   queue: env.MIGRATION_QUEUE,
+ *   concurrency: 20,
+ *   maxRetries: 3,
+ * })
+ * ```
+ */
+export function createBulkMigrationOrchestrator(
+  config: BulkOrchestratorConfig
+): BulkMigrationOrchestrator {
+  return new BulkMigrationOrchestrator(config)
+}