hekenga 0.1.0

data/lib/hekenga/master_process.rb

@@ -0,0 +1,149 @@
+module Hekenga
+  class MasterProcess
+    def initialize(migration)
+      @migration = migration
+    end
+
+    def run!
+      Hekenga.log "Launching migration #{@migration.to_key}"
+      @migration.tasks.each.with_index do |task, idx|
+        launch_task(task, idx)
+        report_while_active(task, idx)
+        if @migration.log(idx).cancel
+          Hekenga.log "TERMINATING DUE TO CRITICAL ERRORS"
+          report_errors(idx)
+          return
+        elsif any_validation_errors?(idx)
+          handle_validation_errors(task, idx)
+          return if @migration.log(idx).cancel
+        end
+        cleanup
+      end
+    end
+
+    def retry!(task_idx, scope)
+      task = @migration.tasks[task_idx]
+      # Reset logs completely
+      Hekenga::Log.where(pkey: @migration.to_key, task_idx: task_idx).delete_all
+      Hekenga::Failure.where(pkey: @migration.to_key, task_idx: task_idx).delete_all
+      @migration.reload_logs
+      # Start the task based on the passed scope - similar to run! but we exit
+      # directly on failure.
+      launch_task(task, task_idx, scope)
+      report_while_active(task, task_idx)
+      if @migration.log(task_idx).cancel
+        return false
+      elsif any_validation_errors?(task_idx)
+        handle_validation_errors(task, task_idx)
+        if @migration.log(task_idx).cancel
+          return false
+        end
+      end
+      cleanup
+      true
+    end
+
+    def any_validation_errors?(idx)
+      Hekenga::Failure::Validation.where(pkey: @migration.to_key, task_idx: idx).any?
+    end
+
+    def handle_validation_errors(task, idx)
+      return unless task.respond_to?(:invalid_strategy)
+      return if idx == @migration.tasks.length - 1
+      case task.invalid_strategy
+      when :prompt
+        unless continue_prompt?("There were validation errors in the last task.")
+          @migration.log(idx).set(cancel: true)
+          return
+        end
+      when :stop
+        Hekenga.log "TERMINATING DUE TO VALIDATION ERRORS"
+        @migration.log(idx).set(cancel: true)
+        return
+      end
+    end
+
+    def report_errors(idx)
+      scope  = @migration.log(idx).failures
+      log_id = @migration.log(idx).id
+      # Validation errors
+      valid_errs     = scope.where(_type: "Hekenga::Failure::Validation")
+      valid_errs_ctr = valid_errs.count
+      unless valid_errs_ctr.zero?
+        Hekenga.log "#{valid_errs_ctr} records failed validation. To get a list:"
+        Hekenga.log "Hekenga::Failure::Validation.lookup('#{log_id}', #{idx})"
+      end
+      # Write failures
+      write_errs     = scope.where(_type: "Hekenga::Failure::Write")
+      write_errs_ctr = write_errs.count
+      unless write_errs_ctr.zero?
+        Hekenga.log "#{write_errs_ctr} write errors detected. Error messages:"
+        Hekenga.log(write_errs.pluck(:message).uniq.map {|x| "- #{x}"}.join("\n"))
+        Hekenga.log "To get a list:"
+        Hekenga.log "Hekenga::Failure::Write.lookup('#{log_id}', #{idx})"
+        # TODO - recover message
+      end
+      # Migration errors
+      general_errs     = scope.where(_type: "Hekenga::Failure::Error")
+      general_errs_ctr = general_errs.count
+      unless general_errs_ctr.zero?
+        Hekenga.log "#{general_errs_ctr} migration errors detected. Error messages:"
+        Hekenga.log(general_errs.pluck(:message).uniq.map {|x| "- #{x}"}.join("\n"))
+        Hekenga.log "To get a list:"
+        Hekenga.log "Hekenga::Failure::Error.lookup('#{log_id}', #{idx})"
+        # TODO - recover message
+      end
+    end
+    def launch_task(task, idx, scope = nil)
+      Hekenga.log "Launching task##{idx}: #{task.description}"
+      @active_thread = Thread.new do
+        @migration.perform_task!(idx, scope)
+      end.tap {|t| t.abort_on_exception = true }
+    end
+    def report_while_active(task, idx)
+      # Wait for the log to be generated
+      until (@migration.log(idx) rescue nil)
+        sleep 1
+      end
+      # Periodically report on thread progress
+      until @migration.log(idx).reload.done
+        @active_thread.join unless @active_thread.alive?
+        report_status(task, idx)
+        return if @migration.log(idx).cancel
+        sleep Hekenga.config.report_sleep
+      end
+      report_status(task, idx)
+      return if @migration.log(idx).cancel
+      report_errors(idx)
+      Hekenga.log "Completed"
+    end
+    def report_status(task, idx)
+      # Simple tasks
+      case task
+      when Hekenga::DocumentTask
+        scope          = @migration.log(idx).failures
+        skipped_ctr    = @migration.log(idx).skipped
+        valid_errs     = scope.where(_type: "Hekenga::Failure::Validation")
+        valid_errs_ctr = valid_errs.count
+        Hekenga.log "Processed #{@migration.log(idx).processed} of #{@migration.log(idx).total} (#{valid_errs_ctr} invalid, #{skipped_ctr} skipped)"
+      when Hekenga::SimpleTask
+        Hekenga.log "Waiting on task"
+      end
+    end
+    def cleanup
+      @active_thread = nil
+    end
+
+    def continue_prompt?(str)
+      loop do
+        print "#{str} Continue? (Y/N)\n"
+        case gets.chomp.downcase
+        when "y"
+          return true
+        when "n"
+          return false
+        end
+      end
+    end
+  end
+end

data/lib/hekenga/migration.rb

@@ -0,0 +1,464 @@
+require 'hekenga/invalid'
+require 'hekenga/context'
+require 'hekenga/parallel_job'
+require 'hekenga/master_process'
+require 'hekenga/log'
+module Hekenga
+  class Migration
+    attr_accessor :stamp, :description, :batch_size
+    attr_reader :tasks
+
+    def initialize
+      @tasks      = []
+      @logs       = {}
+      @batch_size = 25
+    end
+
+    # Internal
+    def timestamp
+      self.stamp.strftime("%Y-%m-%dT%H:%M")
+    end
+
+    def desc_to_token
+      @desc_to_token ||= self.description.gsub(/[^A-Za-z]+/,"_").gsub(/(^_|_$)/,"")
+    end
+
+    def inspect
+      "<Hekenga::Migration #{self.to_key}>"
+    end
+
+    def to_key
+      @pkey ||= "#{timestamp}-#{desc_to_token}"
+    end
+
+    def log(task_idx = @active_idx)
+      raise "Missing task index" if task_idx.nil?
+      @logs[task_idx] ||= Hekenga::Log.find_by(
+        pkey: self.to_key,
+        task_idx: task_idx
+      )
+    end
+
+    def create_log!(attrs = {})
+      @logs[@active_idx] = Hekenga::Log.create(attrs.merge(
+        migration: self,
+        task_idx:  @active_idx
+      ))
+    end
+
+    # API
+    def reload_logs
+      @logs = {}
+    end
+    def performing?
+      Hekenga::Log.where(pkey: self.to_key, done: false).any?
+    end
+    def performed?
+      !!log(self.tasks.length - 1).done
+    end
+    def test_mode!
+      @test_mode = true
+    end
+    def perform!
+      if Hekenga.status(self) == :naught
+        Hekenga::MasterProcess.new(self).run!
+      else
+        Hekenga.log "This migration has already been run! Aborting."
+        return false
+      end
+    end
+    def perform_task!(task_idx = 0, scope = nil)
+      task         = @tasks[task_idx] or return
+      @active_task = task
+      @active_idx  = task_idx
+      case task
+      when Hekenga::SimpleTask
+        start_simple_task(task)
+      when Hekenga::DocumentTask
+        # TODO - online migration support (have log.total update, requeue)
+        scope ||= task.scope.asc(:_id)
+        create_log!(total: scope.count)
+        if task.parallel
+          start_parallel_task(task, task_idx, scope)
+        else
+          start_document_task(task, task_idx, scope)
+        end
+      end
+    end
+    def recover!
+      # NOTE - can't find a way to check this automatically with ActiveJob right now
+      return false unless prompt "Check that the migration queue has processed before recovering. Continue?"
+      # Write failures
+      @tasks.each.with_index do |task, idx|
+        # If no log, run the task now
+        unless Hekenga::Log.where(pkey: self.to_key, task_idx: idx).any?
+          return false unless retry_task!(task, idx)
+          next
+        end
+        # Did this task fail?
+        failedP = log(idx).cancel || Hekenga::Failure.where(pkey: to_key, task_idx: idx).any?
+        # If it didn't, keep searching
+        next unless failedP
+        # This is the first failure we've detected - recover from it
+        case task
+        when Hekenga::DocumentTask
+          ret = recover_document_task!(task, idx)
+        when Hekenga::SimpleTask
+          ret = recover_simple!(task, idx)
+        end
+
+        case ret
+        when :next
+          next
+        when :cancel
+          return false
+        else
+          return false unless retry_task!(task, idx, ret)
+        end
+      end
+      return true
+    end
+
+    def retry_task!(task, idx, scope = nil)
+      Hekenga.log "Retrying task##{idx}"
+      unless Hekenga::MasterProcess.new(self).retry!(idx, scope)
+        Hekenga.log "Failed to retry the task. Aborting.."
+        return false
+      end
+      return true
+    end
+
+    def recover_simple!(task, idx)
+      # Simple tasks just get retried - no fuss
+      Hekenga.log("Found failed simple task. Retrying..")
+      return
+    end
+
+    def recover_document_task!(task, idx)
+      # Document tasks are a bit more involved.
+      validation_failures = Hekenga::Failure::Validation.where(pkey: to_key, task_idx: idx)
+      write_failures      = Hekenga::Failure::Write.where(pkey: to_key, task_idx: idx)
+      error_failures      = Hekenga::Failure::Error.where(pkey: to_key, task_idx: idx)
+      cancelled_failures  = Hekenga::Failure::Cancelled.where(pkey: to_key, task_idx: idx)
+
+      # Stats
+      validation_failure_ctr = validation_failures.count
+      write_failure_ctr      = write_failures.count
+      error_failure_ctr      = error_failures.count
+      cancelled_failure_ctr  = cancelled_failures.count
+
+      # Prompt for recovery
+      recoverP = prompt(
+        "Found #{validation_failure_ctr} invalid, "+
+        "#{write_failure_ctr} failed writes, "+
+        "#{error_failure_ctr} errors, "+
+        "#{cancelled_failure_ctr} cancelled on migration. Recover?"
+      )
+      return :next unless recoverP
+
+      # Recover from critical write failures (DB records potentially lost)
+      unless write_failure_ctr.zero?
+        Hekenga.log "Recovering old data from #{write_failure_ctr} write failure(s)"
+        recover_data(write_failures, task.scope.klass)
+      end
+
+      # Resume task from point of error
+      if task.parallel
+        # TODO - support for recovery on huge # IDs
+        failed_ids = [
+          write_failures.pluck(:document_ids),
+          error_failures.pluck(:batch_start),
+          cancelled_failures.pluck(:document_ids),
+          validation_failures.pluck(:doc_id)
+        ].flatten.compact
+        resume_scope = task.scope.klass.asc(:_id).in(_id: failed_ids)
+      else
+        first_id     = error_failures.first&.batch_start || write_failures.first&.batch_start
+        invalid_ids  = validation_failures.pluck(:doc_id)
+        if first_id && invalid_ids.any?
+          resume_scope = task.scope.klass.asc(:_id).and(
+            task.scope.selector,
+            task.scope.klass.or(
+              {_id: {:$gte => first_id}},
+              {_id: {:$in  => invalid_ids}}
+            ).selector
+          )
+        elsif first_id
+          resume_scope = task.scope.asc(:_id).gte(_id: first_id)
+        elsif invalid_ids.any?
+          resume_scope = task.scope.klass.asc(:_id).in(_id: invalid_ids)
+        else
+          resume_scope = :next
+        end
+      end
+
+      return resume_scope
+    end
+
+    def recover_data(write_failures, klass)
+      write_failures.each do |write_failure|
+        failed_ids = write_failure.document_ids
+        extant     = klass.in(_id: failed_ids).pluck(:_id)
+        to_recover = (failed_ids - extant)
+        docs       = write_failure.documents.find_all {|x| to_recover.include?(x["_id"])}
+        next if docs.empty?
+        Hekenga.log "Recovering #{docs.length} documents.."
+        klass.collection.insert_many(docs)
+      end
+    end
+
+    def prompt(str)
+      loop do
+        print "#{str} (Y/N):\n"
+        case gets.chomp.downcase
+        when "y"
+          return true
+        when "n"
+          return false
+        end
+      end
+    end
+
+    def rollback!
+      # TODO
+    end
+
+    # Internal perform methods
+    def start_simple_task(task)
+      create_log!
+      begin
+        with_setup do
+          task.up!(@context)
+        end
+      rescue => e
+        simple_failure!(e)
+        return
+      end
+      log_done!
+    end
+
+    def check_for_completion
+      if log.processed == log.total
+        log_done!
+      end
+    end
+    def log_done!
+      log.set(done: true, finished: Time.now)
+    end
+    def start_parallel_task(task, task_idx, scope)
+      # TODO - support for crazy numbers of documents where pluck is too big
+      scope.asc(:_id).pluck(:_id).take(log.total).each_slice(batch_size).each do |ids|
+        Hekenga::ParallelJob.perform_later(
+          self.to_key, task_idx, ids.map(&:to_s), !!@test_mode
+        )
+      end
+    end
+    def run_parallel_task(task_idx, ids)
+      if log(task_idx).cancel
+        failed_cancelled!(ids)
+        return
+      end
+      task = self.tasks[task_idx] or return
+      @active_idx = task_idx
+      with_setup(task) do
+        process_batch(task, task.scope.asc(:_id).in(_id: ids).to_a)
+        unless @skipped.empty?
+          failed_cancelled!(@skipped)
+        end
+      end
+    end
+    def with_setup(task = nil)
+      @context = Hekenga::Context.new(@test_mode)
+      task&.setups&.each do |block|
+        @context.instance_exec(&block)
+      end
+      # Disable specific callbacks
+      begin
+        task&.disable_rules&.each do |rule|
+          rule[:klass].skip_callback rule[:callback]
+        end
+        yield
+      ensure
+        @context = nil
+        # Make sure the callbacks make it back
+        task&.disable_rules&.each do |rule|
+          rule[:klass].set_callback rule[:callback]
+        end
+      end
+    end
+    def start_document_task(task, task_idx, scope)
+      records = []
+      with_setup(task) do
+        scope.asc(:_id).each do |record|
+          records.push(record)
+          if records.length == batch_size
+            process_batch(task, records)
+            return if log.cancel
+            records = []
+          end
+        end
+        process_batch(task, records) if records.any?
+      end
+      log_done!
+    end
+    def run_filters(task, record)
+      task.filters.all? do |block|
+        @context.instance_exec(record, &block)
+      end
+    end
+    def process_batch(task, records)
+      @skipped   = []
+      to_persist = []
+      fallbacks  = []
+
+      filtered = records.group_by do |record|
+        run_filters(task, record)
+      end
+      log_skipped(task, filtered[false]) if filtered[false]
+      return unless filtered[true]
+      filtered[true].map.with_index do |record, idx|
+        original_record = Marshal.load(Marshal.dump(record.as_document))
+        begin
+          task.up!(@context, record)
+        rescue => e
+          failed_apply!(e, record, records[0].id)
+          @skipped = filtered[true][idx+1..-1]
+          return
+        end
+        if validate_record(record)
+          to_persist.push(record)
+          fallbacks.push(original_record)
+        else
+          if log.cancel
+            @skipped = filtered[true][idx+1..-1]
+            return
+          end
+        end
+      end.compact
+      persist_batch(task, to_persist, fallbacks)
+    end
+    def log_skipped(task, records)
+      log.incr_and_return(
+        skipped:   records.length,
+        processed: records.length
+      )
+      check_for_completion
+    end
+    def log_success(task, records)
+      log.incr_and_return(
+        processed: records.length
+      )
+      check_for_completion
+    end
+
+    def persist_batch(task, records, original_records)
+      if @test_mode
+        log_success(task, records)
+        return
+      end
+      # NOTE - edgecase where callbacks cause the record to become invalid is
+      # not covered
+      records.each do |record|
+        begin
+          next if task.skip_prepare
+          if task.timeless
+            record.timeless.send(:prepare_update) {}
+          else
+            record.send(:prepare_update) {}
+          end
+        rescue => e
+          # If prepare_update throws an error, we're in trouble - crash out now
+          failed_apply!(e, record, records[0].id)
+          return
+        end
+      end
+      begin
+        delete_records!(task.scope.klass, records.map(&:_id))
+        write_records!(task.scope.klass, records)
+        log_success(task, records)
+      rescue => e
+        failed_write!(e, original_records)
+      end
+    end
+    def delete_records!(klass, ids)
+      klass.in(_id: ids).delete_all
+    end
+    def write_records!(klass, records)
+      klass.collection.insert_many(records.map(&:as_document))
+    end
+    def simple_failure!(error)
+      log.add_failure({
+        message:   error.to_s,
+        backtrace: error.backtrace,
+        simple:    true
+      }, Hekenga::Failure::Error)
+      log_cancel!
+    end
+    def failed_cancelled!(ids)
+      log.add_failure({
+        document_ids: ids,
+        batch_start: ids[0]
+      }, Hekenga::Failure::Cancelled)
+    end
+    def failed_apply!(error, record, batch_start_id)
+      log.add_failure({
+        message:     error.to_s,
+        backtrace:   error.backtrace,
+        document:    Marshal.load(Marshal.dump(record.as_document)),
+        batch_start: batch_start_id
+      }, Hekenga::Failure::Error)
+      log_cancel!
+    end
+    def log_cancel!
+      log.set(cancel: true, error: true, done: true, finished: Time.now)
+    end
+    def failed_write!(error, original_records)
+      log.add_failure({
+        message:      error.to_s,
+        backtrace:    error.backtrace,
+        documents:    original_records,
+        document_ids: original_records.map {|x| x["_id"]},
+        batch_start:  original_records[0]["_id"]
+      }, Hekenga::Failure::Write)
+      log_cancel!
+    end
+    def failed_validation!(record)
+      log.add_failure({
+        doc_id:   record.id,
+        errs:     record.errors.full_messages,
+        document: Marshal.load(Marshal.dump(record.as_document))
+      }, Hekenga::Failure::Validation)
+      log.set(error: true)
+      log.incr_and_return(processed: 1, unvalid: 1)
+      if @active_task.invalid_strategy == :cancel
+        log_cancel!
+      else
+        check_for_completion
+      end
+    end
+    def validate_record(record)
+      # TODO - ability to skip validation
+      if record.valid?
+        true
+      else
+        failed_validation!(record)
+        false
+      end
+    end
+
+    # Validations
+    MIN_TOKEN_LENGTH = 5
+
+    def validation_error(field, reason)
+      raise Hekenga::Invalid.new(self, field, reason)
+    end
+
+    def validate!
+      validation_error(:stamp,       "missing")   unless self.stamp.is_a?(Time)
+      validation_error(:description, "missing")   unless self.description
+      validation_error(:description, "too short") unless self.desc_to_token.length > 5
+      validation_error(:tasks,       "missing")   if self.tasks.length.zero?
+      true
+    end
+  end
+end

data/lib/hekenga/parallel_job.rb

@@ -0,0 +1,11 @@
+require 'active_job'
+module Hekenga
+  class ParallelJob < ActiveJob::Base
+    queue_as :migration
+    def perform(migration_key, task_idx, ids, test_mode)
+      migration = Hekenga.find_migration(migration_key)
+      migration.test_mode! if test_mode
+      migration.run_parallel_task(task_idx, ids)
+    end
+  end
+end

data/lib/hekenga/simple_task.rb

@@ -0,0 +1,32 @@
+require 'hekenga/irreversible'
+module Hekenga
+  class SimpleTask
+    attr_reader :ups, :downs
+    attr_accessor :description
+    def initialize
+      @ups   = []
+      @downs = []
+    end
+
+    def validate!
+      raise Hekenga::Invalid.new(self, :ups, "missing") unless ups.any?
+    end
+
+    def up!(context)
+      @ups.each do |block|
+        context.instance_exec(&block)
+      end
+    end
+
+    def down!
+      raise Hekenga::Irreversible.new(self) unless reversible?
+      @downs.each do |block|
+        block.call
+      end
+    end
+
+    def reversible?
+      downs.any?
+    end
+  end
+end

data/lib/hekenga/version.rb

@@ -0,0 +1,3 @@
+module Hekenga
+  VERSION = "0.1.0"
+end

data/lib/hekenga/virtual_method.rb

@@ -0,0 +1,8 @@
+require 'hekenga/base_error'
+module Hekenga
+  class VirtualMethod < Hekenga::BaseError
+    def initialize(klass, method)
+      super("#{klass.to_s}##{method} has not been implemented.")
+    end
+  end
+end