hekenga 0.2.13 → 1.0.0

data/lib/hekenga/log.rb

@@ -18,12 +18,7 @@ module Hekenga
     field :skip,   default: false
 
     # Used by document tasks
-    field :total
-    field :processed, default: 0
-    field :skipped,   default: 0
-    field :unvalid,   default: 0
-    field :started,   default: ->{ Time.now }
-    field :finished,  type:    Time
+    field :finished, type: Time
 
     has_many :failures, class_name: "Hekenga::Failure"
 
@@ -36,21 +31,21 @@ module Hekenga
     end
 
     def add_failure(attrs, klass)
-      self.failures.create({
-        pkey:     self.pkey,
-        task_idx: self.task_idx
-      }.merge(attrs), klass)
+      failure = klass.new(attrs.merge(pkey: pkey, task_idx: task_idx, log_id: _id))
+      failure.send(:prepare_insert) {}
+      Hekenga::Failure.collection.insert_one(
+        failure.as_document,
+        session: nil
+      )
     end
 
-    def incr_and_return(fields)
-      doc = self.class.where(_id: self.id).find_one_and_update({
-        :$inc => fields
-      }, return_document: :after, projection: fields.keys.map {|x| [x, 1]}.to_h, upsert: true)
-      fields.map do |field, _|
-        value = doc.send(field)
-        send("#{field}=", value)
-        [field, value]
-      end.to_h
+    def set_without_session(attrs)
+      self.class.collection.update_one(
+        { _id: _id },
+        {'$set': attrs},
+        session: nil
+      )
+      self.attributes = attrs
     end
   end
 end

data/lib/hekenga/master_process.rb

@@ -1,3 +1,6 @@
+require 'hekenga/task_failed_error'
+require 'hekenga/task_splitter'
+
 module Hekenga
   class MasterProcess
     def initialize(migration)
@@ -5,101 +8,33 @@ module Hekenga
     end
 
     def run!
-      Hekenga.log "Launching migration #{@migration.to_key}"
+      Hekenga.log "Launching migration #{@migration.to_key}: #{@migration.description}"
       @migration.tasks.each.with_index do |task, idx|
         launch_task(task, idx)
         report_while_active(task, idx)
-        if @migration.log(idx).cancel
-          Hekenga.log "TERMINATING DUE TO CRITICAL ERRORS"
-          report_errors(idx)
-          return
-        elsif any_validation_errors?(idx)
-          handle_validation_errors(task, idx)
-          return if @migration.log(idx).cancel
-        end
-        cleanup
+      rescue Hekenga::TaskFailedError
+        return false
+      ensure
+        @active_thread = nil
       end
+      true
     end
 
-    def retry!(task_idx, scope)
-      task = @migration.tasks[task_idx]
-      # Reset logs completely
-      Hekenga::Log.where(pkey: @migration.to_key, task_idx: task_idx).delete_all
-      Hekenga::Failure.where(pkey: @migration.to_key, task_idx: task_idx).delete_all
-      @migration.reload_logs
-      # Start the task based on the passed scope - similar to run! but we exit
-      # directly on failure.
-      launch_task(task, task_idx, scope)
-      report_while_active(task, task_idx)
-      if @migration.log(task_idx).cancel
+    def recover!
+      Hekenga.log "Recovering migration #{@migration.to_key}: #{@migration.description}"
+      @migration.tasks.each.with_index do |task, idx|
+        recover_task(task, idx)
+        report_while_active(task, idx) if @active_thread
+      rescue Hekenga::TaskFailedError
         return false
-      elsif any_validation_errors?(task_idx)
-        handle_validation_errors(task, task_idx)
-        if @migration.log(task_idx).cancel
-          return false
-        end
+      ensure
+        @active_thread = nil
       end
-      cleanup
       true
     end
 
-    def any_validation_errors?(idx)
-      Hekenga::Failure::Validation.where(pkey: @migration.to_key, task_idx: idx).any?
-    end
-
-    def handle_validation_errors(task, idx)
-      return unless task.respond_to?(:invalid_strategy)
-      return if idx == @migration.tasks.length - 1
-      case task.invalid_strategy
-      when :prompt
-        unless continue_prompt?("There were validation errors in the last task.")
-          @migration.log(idx).set(cancel: true)
-          return
-        end
-      when :stop
-        Hekenga.log "TERMINATING DUE TO VALIDATION ERRORS"
-        @migration.log(idx).set(cancel: true)
-        return
-      end
-    end
+    private
 
-    def report_errors(idx)
-      scope  = @migration.log(idx).failures
-      log_id = @migration.log(idx).id
-      # Validation errors
-      valid_errs     = scope.where(_type: "Hekenga::Failure::Validation")
-      valid_errs_ctr = valid_errs.count
-      unless valid_errs_ctr.zero?
-        Hekenga.log "#{valid_errs_ctr} records failed validation. To get a list:"
-        Hekenga.log "Hekenga::Failure::Validation.lookup('#{log_id}', #{idx})"
-      end
-      # Write failures
-      write_errs     = scope.where(_type: "Hekenga::Failure::Write")
-      write_errs_ctr = write_errs.count
-      unless write_errs_ctr.zero?
-        Hekenga.log "#{write_errs_ctr} write errors detected. Error messages:"
-        Hekenga.log(write_errs.pluck(:message).uniq.map {|x| "- #{x}"}.join("\n"))
-        Hekenga.log "To get a list:"
-        Hekenga.log "Hekenga::Failure::Write.lookup('#{log_id}', #{idx})"
-        # TODO - recover message
-      end
-      # Migration errors
-      general_errs     = scope.where(_type: "Hekenga::Failure::Error")
-      general_errs_ctr = general_errs.count
-      unless general_errs_ctr.zero?
-        Hekenga.log "#{general_errs_ctr} migration errors detected. Error messages:"
-        Hekenga.log(general_errs.pluck(:message).uniq.map {|x| "- #{x}"}.join("\n"))
-        Hekenga.log "To get a list:"
-        Hekenga.log "Hekenga::Failure::Error.lookup('#{log_id}', #{idx})"
-        # TODO - recover message
-      end
-    end
-    def launch_task(task, idx, scope = nil)
-      Hekenga.log "Launching task##{idx}: #{task.description}"
-      @active_thread = Thread.new do
-        @migration.perform_task!(idx, scope)
-      end.tap {|t| t.abort_on_exception = true }
-    end
     def report_while_active(task, idx)
       # Wait for the log to be generated
       until (@migration.log(idx) rescue nil)
@@ -107,43 +42,187 @@ module Hekenga
       end
       # Periodically report on thread progress
       until @migration.log(idx).reload.done
-        @active_thread.join unless @active_thread.alive?
+        @active_thread.join
         report_status(task, idx)
-        return if @migration.log(idx).cancel
         sleep Hekenga.config.report_sleep
       end
-      report_status(task, idx)
-      return if @migration.log(idx).cancel
-      report_errors(idx)
+      report_status(task, idx) if task.is_a?(Hekenga::DocumentTask)
+      report_result(task, idx)
       Hekenga.log "Completed"
     end
+
+    def recover_task(task, idx)
+      case task
+      when Hekenga::DocumentTask
+        recover_document_task(task, idx)
+      when Hekenga::SimpleTask
+        recover_simple_task(task, idx)
+      end
+    end
+
+    def recover_document_task(task, idx)
+      log = @migration.log(idx) rescue nil
+      if log.nil?
+        launch_task(task, idx)
+      elsif document_task_failed?(log, idx, fail_on_invalid: true)
+        Hekenga.log "Recovering task##{idx}: #{task.description}"
+        log.set_without_session({
+          done: false,
+          error: false,
+          cancel: false,
+          finished: nil,
+        })
+        recover_write_failures(task, log)
+        task_records = @migration.task_records(idx)
+        if task.parallel
+          in_thread do
+            Hekenga::ParallelTask.new(
+              migration: @migration,
+              task: task,
+              task_idx: idx,
+              test_mode: @migration.test_mode
+            ).resume!
+          end
+        else
+          # Strategy: clear failures; reset state
+          log.failures.delete_all
+          task_records.incomplete.delete_all
+          task_records.each do |record|
+            Hekenga::TaskSplitter.new(record, @executor_key).call&.destroy
+          end
+          @migration.active_idx = idx
+          in_thread do
+            @migration.start_document_task(task, idx, recover: true)
+          end
+        end
+
+      else
+        Hekenga.log "Skipping completed task##{idx}: #{task.description}"
+      end
+    end
+
+    def document_task_failed?(log, idx, fail_on_invalid:)
+      return true if log.nil?
+      return true if log.error
+      return true if log.cancel
+      return true if @migration.task_records(idx).incomplete.any?
+
+      stats = combined_stats(idx)
+      return false if stats.blank?
+      return true if stats['failed'].positive?
+      return true if fail_on_invalid && stats['invalid'].positive?
+
+      false
+    end
+
+    def recover_write_failures(task, log)
+      klass = task.scope.klass
+      log.failures.where(_type: "Hekenga::Failure::Write").each do |write_failure|
+        next unless write_failure.documents.any?
+
+        existing = klass.in(_id: write_failure.documents.map {|doc| doc["_id"]}).pluck(:_id).to_set
+        to_write = write_failure.documents.reject {|doc| existing.include?(doc["_id"])}
+        next if to_write.empty?
+
+        Hekenga.log("Recovering #{to_write.length} write failures")
+        klass.collection.insert_many(to_write)
+      end.delete_all
+    end
+
+    def recover_simple_task(task, idx)
+      log = @migration.log(idx) rescue nil
+      if log.nil?
+        Hekenga.log "Recovering task##{idx}: #{task.description}"
+        launch_task(task, idx)
+      elsif log.error
+        Hekenga.log "Recovering task##{idx}: #{task.description}"
+        # Strategy: clear logs + rerun
+        log.failures.delete_all
+        log.destroy
+        @migration.reload_logs
+        launch_task(task, idx)
+      else
+        Hekenga.log "Skipping completed task##{idx}: #{task.description}"
+      end
+    end
+
+    def launch_task(task, idx)
+      Hekenga.log "Launching task##{idx}: #{task.description}"
+      in_thread do
+        @migration.perform_task!(idx)
+      end
+    end
+
+    def in_thread(&block)
+      @active_thread = Thread.new(&block).tap do |t|
+        t.report_on_exception = false
+        t.abort_on_exception = true
+      end
+    end
+
     def report_status(task, idx)
-      # Simple tasks
       case task
       when Hekenga::DocumentTask
-        scope          = @migration.log(idx).failures
-        skipped_ctr    = @migration.log(idx).skipped
-        valid_errs     = scope.where(_type: "Hekenga::Failure::Validation")
-        valid_errs_ctr = valid_errs.count
-        Hekenga.log "Processed #{@migration.log(idx).processed} of #{@migration.log(idx).total} (#{valid_errs_ctr} invalid, #{skipped_ctr} skipped)"
+        if task.parallel
+          Hekenga.log "#{@migration.task_records(idx).complete.count} of #{@migration.task_records(idx).count} batches processed"
+          check_for_completion!(idx)
+        else
+          Hekenga.log "#{@migration.task_records(idx).complete.count} batches processed"
+        end
       when Hekenga::SimpleTask
-        Hekenga.log "Waiting on task"
+        Hekenga.log "Waiting for task to complete"
       end
     end
-    def cleanup
-      @active_thread = nil
+
+    def report_result(task, idx)
+      case task
+      when Hekenga::DocumentTask
+        Hekenga.log "Migration result:"
+        combined_stats(idx)&.each do |stat, count|
+          Hekenga.log " - #{stat.capitalize}: #{count}"
+        end
+        if document_task_failed?(@migration.log(idx), idx, fail_on_invalid: false)
+          Hekenga.log "There were failures while running the task. Stopping"
+          raise Hekenga::TaskFailedError
+        end
+      when Hekenga::SimpleTask
+        report_simple_result(idx)
+      end
+    end
+
+    def combined_stats(idx)
+      Hekenga::DocumentTaskRecord.collection.aggregate([
+        { "$match" => @migration.task_records(idx).selector },
+        { "$group" => {
+          "_id" => "1",
+          "failed" => { "$sum" => "$stats.failed" },
+          "invalid" => { "$sum" => "$stats.invalid" },
+          "written" => { "$sum" => "$stats.written" },
+        }}
+      ]).to_a[0]&.except("_id")
     end
 
-    def continue_prompt?(str)
-      loop do
-        print "#{str} Continue? (Y/N)\n"
-        case gets.chomp.downcase
-        when "y"
-          return true
-        when "n"
-          return false
+    def report_simple_result(idx)
+      if @migration.log(idx).failures.any?
+        Hekenga.log "The task crashed with the following error message:"
+        @migration.log(idx).failures.each do |failure|
+          Hekenga.log failure.message
         end
+        raise Hekenga::TaskFailedError
+      else
+        Hekenga.log "Task succeeded"
       end
     end
+
+    def check_for_completion!(idx)
+      complete = @migration.task_records(idx).incomplete.none?
+      return unless complete
+
+      @migration.log(idx).set_without_session(
+        done: true,
+        finished: Time.now,
+        error: @migration.task_records(idx).failed.any?
+      )
+    end
   end
 end