libis-workflow 2.0.beta.13 → 2.0.beta.14

data/lib/libis/workflow/task.rb

@@ -15,69 +15,82 @@ module Libis
       include ::Libis::Workflow::Base::Logger
       include ::Libis::Tools::ParameterContainer
 
-      attr_accessor :parent, :name, :workitem, :tasks
+      attr_accessor :parent, :name, :workitem
 
       parameter quiet: false, description: 'Prevemt generating log output.'
-      parameter abort_on_error: false, description: 'Stop all tasks when an error occurs.'
-      parameter always_run: false, description: 'Run this task, even if the item failed a previous task.'
-      parameter subitems: false, description: 'Do not process the given item, but only the subitems.'
       parameter recursive: false, description: 'Run the task on all subitems recursively.'
+      parameter retry_count: 0, description: 'Number of times to retry the task.'
+      parameter retry_interval: 10, description: 'Number of seconds to wait between retries.'
 
       def self.task_classes
-        ObjectSpace.each_object(::Class).select {|klass| klass < self}
+        ObjectSpace.each_object(::Class).select { |klass| klass < self }
       end
 
       def initialize(parent, cfg = {})
         @subitems_stopper = false
         @subtasks_stopper = false
         self.parent = parent
-        self.tasks = []
         configure cfg
       end
 
       def <<(task)
-        self.tasks << task
+        raise Libis::WorkflowError, "Processing task '#{self.namepath}' is not allowed to have subtasks."
       end
 
+      # @param [Libis::Workflow::Base::WorkItem] item
       def run(item)
-
         check_item_type ::Libis::Workflow::Base::WorkItem, item
+        self.workitem = item
 
-        return if item.failed? unless parameter(:always_run)
+        case self.action
+          when :retry
+            return if item.check_status(:DONE, self.namepath)
+          when :failed
+            return
+          else
+        end
+
+        (parameter(:retry_count)+1).times do
 
-        if parameter(:subitems)
-            log_started item
-            run_subitems item
-            log_done(item) unless item.failed?
-        else
           run_item(item)
-        end
 
-        item.save
+          case item.status(self.namepath)
+            when :DONE
+              self.action = :run
+              return
+            when :ASYNC_WAIT
+              self.action = :retry
+            when :ASYNC_HALT
+              break
+            when :FAILED
+              break
+            else
+              return
+          end
 
-      end
+          self.action = :retry
 
-      def run_item(item)
+          sleep(parameter(:retry_interval))
 
-        begin
+        end
 
-          self.workitem = item
+        item.get_run.action = :failed
 
-          process_item item
+      rescue WorkflowError => e
+        error e.message, item
+        update_status item, :FAILED
 
-        rescue WorkflowError => e
-          error e.message
-          log_failed item
+      rescue WorkflowAbort => e
+        update_status item, :FAILED
+        raise e if parent
 
-        rescue WorkflowAbort => e
-          item.status = to_status :failed
-          raise e if parent
+      rescue ::Exception => e
+        update_status item, :FAILED
+        fatal "Exception occured: #{e.message}", item
+        debug e.backtrace.join("\n")
 
-        rescue ::Exception => e
-          fatal 'Exception occured: %s', e.message
-          debug e.backtrace.join("\n")
-          log_failed item
-        end
+      ensure
+        item.save
 
       end
 
@@ -85,7 +98,9 @@ module Libis
         (self.parent.names rescue Array.new).push(name).compact
       end
 
-      def namepath; self.names.join('/'); end
+      def namepath;
+        self.names.join('/');
+      end
 
       def apply_options(opts)
         o = {}
@@ -103,74 +118,35 @@ module Libis
           end
         end
 
-        self.tasks.each do |task|
-          task.apply_options opts
-        end
       end
 
       protected
 
-      def stop_processing_subitems
-        @subitems_stopper = true if parameter(:recursive)
-      end
-
-      def check_processing_subitems
-        if @subitems_stopper
-          @subitems_stopper = false
-          return false
+      def configure(cfg)
+        self.name = cfg[:name] || (cfg[:class] || self.class).to_s.split('::').last
+        (cfg[:options] || {}).merge(
+            cfg.reject { |k, _| [:options, :name, :class].include? k.to_sym }
+        ).symbolize_keys.each do |k, v|
+          self.parameter(k, v)
         end
-        true
       end
 
-      def stop_processing_subtasks
-        @subtasks_stopper= true
-      end
+      def run_item(item)
+        @item_skipper = false
 
-      def check_processing_subtasks
-        if @subtasks_stopper
-          @subtasks_stopper = false
-          return false
-        end
-        true
-      end
+        pre_process(item)
 
-      def skip_processing_item
-        @item_skipper = true
-      end
+        set_status item, :STARTED
 
-      def log_started(item)
-        item.status = to_status :started
-      end
+        self.process item unless @item_skipper
 
-      def log_failed(item, message = nil)
-        warn (message), item if message
-        item.status = to_status :failed
-      end
+        run_subitems(item) if parameter(:recursive)
 
-      def log_done(item)
-        item.status = to_status :done
-      end
+        update_status item, :DONE
 
-      def process_item(item)
-        @item_skipper = false
-        pre_process(item)
-        unless @item_skipper
-          log_started item
-          process item
-        end
-        run_subitems(item) if parameter(:recursive)
-        unless @item_skipper
-          run_subtasks item
-          log_done item unless item.failed?
-        end
         post_process item
       end
 
-      def process(item)
-        # needs implementation unless there are subtasks
-        raise RuntimeError, 'Should be overwritten' if self.tasks.empty?
-      end
-
       def pre_process(_)
         true
         # optional implementation
@@ -180,12 +156,39 @@ module Libis
         # optional implementation
       end
 
-      def get_root_item
-        self.workitem.root
+      def run_subitems(parent_item)
+        return unless check_processing_subitems
+        items = subitems parent_item
+        return unless items.count > 0
+
+        status = Hash.new(0)
+        items.each_with_index do |item, i|
+          debug 'Processing subitem (%d/%d): %s', parent_item, i+1, items.count, item.to_s
+          run_item item
+          status[item.status(self.namepath)] += 1
+        end
+
+        debug '%d of %d subitems passed', parent_item, status[:DONE], items.count
+        substatus_check(status, parent_item, 'item')
       end
 
-      def get_work_dir
-        get_root_item.work_dir
+      def substatus_check(status, item, task_or_item)
+        if (failed = status[:FAILED] > 0)
+          warn "%d sub#{task_or_item}(s) failed", item, failed
+          update_status(item, :FAILED)
+        end
+
+        if (halted = status[:ASYNC_HALT] > 0)
+          warn "%d sub#{task_or_item}(s) halted in async process", item, halted
+          update_status(item, :ASYNC_HALT)
+        end
+
+        if (waiting = status[:ASYNC_WAIT] > 0)
+          warn "waiting for %d sub#{task_or_item}(s) in async process", item, waiting
+          update_status(item, :ASYNC_WAIT)
+        end
+
+        update_status(item, :DONE)
       end
 
       def capture_cmd(cmd, *opts)
@@ -200,61 +203,50 @@ module Libis
         $stderr = STDERR
       end
 
-      def run_subitems(parent_item)
-        return unless check_processing_subitems
-        items = subitems parent_item
-        failed = passed = 0
-        items.each_with_index do |item, i|
-          debug 'Processing subitem (%d/%d): %s', parent_item, i+1, items.count, item.to_s
-          run_item item
-          if item.failed?
-            failed += 1
-            if parameter(:abort_on_error)
-              error 'Aborting ...', parent_item
-              raise WorkflowAbort.new "Aborting: task #{name} failed on #{item}"
-            end
-          else
-            passed += 1
-          end
-        end
-        if failed > 0
-          warn '%d subitem(s) failed', parent_item, failed
-          if failed == items.count
-            error 'All subitems have failed', parent_item
-            log_failed parent_item
-            return
-          end
-        end
-        debug '%d of %d subitems passed', parent_item, passed, items.count if items.count > 0
+      def action=(action)
+        self.workitem.get_run.action = action
       end
 
-      def run_subtasks(item)
-        return unless check_processing_subtasks
-        tasks = subtasks item
-        tasks.each_with_index do |task, i|
-          debug 'Running subtask (%d/%d): %s', item, i+1, tasks.count, task.name
-          task.run item
-          if item.failed?
-            if task.parameter(:abort_on_error)
-              error 'Aborting ...'
-              raise WorkflowAbort.new "Aborting: task #{task.name} failed on #{item}"
-            end
-            return
-          end
-        end
+      def action
+        self.workitem.get_run.action
       end
 
-      def configure(cfg)
-        self.name = cfg[:name] || (cfg[:class] || self.class).to_s.split('::').last
-        (cfg[:options] || {}).merge(
-            cfg.reject { |k, _| [:options, :name, :class].include? k.to_sym }
-        ).symbolize_keys.each do |k,v|
-          self.parameter(k,v)
+      def get_root_item(item = nil)
+        (item || self.workitem).get_root
+      end
+
+      def get_work_dir(item = nil)
+        get_root_item(item).work_dir
+      end
+
+      def stop_processing_subitems
+        @subitems_stopper = true if parameter(:recursive)
+      end
+
+      def check_processing_subitems
+        if @subitems_stopper
+          @subitems_stopper = false
+          return false
         end
+        true
+      end
+
+      def skip_processing_item
+        @item_skipper = true
+      end
+
+      def update_status(item, state)
+        return nil unless item.compare_status(state, self.namepath) < 0
+        set_status(item, state)
+      end
+
+      def set_status(item, state)
+        item.status = to_status(state)
+        state
       end
 
-      def to_status(text)
-        [text.to_s.capitalize, self.names]
+      def to_status(state)
+        [state, self.namepath]
       end
 
       def check_item_type(klass, item = nil)
@@ -271,16 +263,12 @@ module Libis
 
       private
 
-      def subtasks(item = nil)
-        self.tasks.map do |task|
-          ((item || self.workitem).failed? and not task.parameter(:always_run)) ? nil : task
-        end.compact
+      def subtasks
+        self.tasks
       end
 
       def subitems(item = nil)
-        items = (item || workitem).items
-        return items if self.parameter(:always_run)
-        items.reject { |i| i.failed? }
+        (item || workitem).items
       end
 
       def default_values
@@ -288,7 +276,7 @@ module Libis
       end
 
       def self.default_values
-        parameter_defs.inject({}) do |hash,parameter|
+        parameter_defs.inject({}) do |hash, parameter|
           hash[parameter.first] = parameter.last[:default]
           hash
         end

data/lib/libis/workflow/task_group.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require_relative 'task'
+
+module Libis
+  module Workflow
+
+    # noinspection RubyTooManyMethodsInspection
+    class TaskGroup < Libis::Workflow::Task
+
+      attr_accessor :tasks
+
+      def initialize(parent, cfg = {})
+        self.tasks = []
+        super parent, cfg
+      end
+
+      def <<(task)
+        self.tasks << task
+      end
+
+      def apply_options(opts)
+        super opts
+        self.tasks.each do |task|
+          task.apply_options opts
+        end
+      end
+
+      protected
+
+      def process(item)
+
+        return unless check_processing_subtasks
+
+        tasks = subtasks
+        return unless tasks.count > 0
+
+        status = Hash.new(0)
+        tasks.each_with_index do |task, i|
+          debug 'Running subtask (%d/%d): %s', item, i+1, tasks.count, task.name
+          task.run item
+          status[item.status(task.namepath)] += 1
+        end
+
+        substatus_check(status, item, 'task')
+      end
+
+      def stop_processing_subtasks
+        @subtasks_stopper= true
+      end
+
+      def check_processing_subtasks
+        if @subtasks_stopper
+          @subtasks_stopper = false
+          return false
+        end
+        true
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/tasks/analyzer.rb

@@ -9,14 +9,12 @@ module Libis
       class Analyzer < Task
 
         parameter quiet: true, frozen: true
-        parameter abort_on_error: false, frozen: true
-        parameter always_run: true, frozen: true
-        parameter subitems: false, frozen: true
         parameter recursive: false, frozen: true
 
+        # @param [Libis::Workflow::Base::WorkItem] item
         def run(item)
 
-          item.properties[:ingest_failed] = item.failed?
+          item.properties[:ingest_failed] = item.check_status(:FAILED)
 
           item.summary = {}
           item.log_history.each do |log|

data/lib/libis/workflow/version.rb

@@ -2,6 +2,6 @@
 
 module Libis
   module Workflow
-    VERSION = '2.0.beta.13' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
+    VERSION = '2.0.beta.14' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
   end
 end

data/lib/libis/workflow/work_item.rb

@@ -32,17 +32,9 @@ module Libis
         self.log_history << msg.merge(c_at: ::Time.now)
       end
 
-      def add_status_log(message, tasklist = nil)
+      def add_status_log(info)
         # noinspection RubyResolve
-        self.status_log << {
-            timestamp: ::Time.now,
-            tasklist: tasklist,
-            text: message
-        }.cleanup
-      end
-
-      def status_label(status_entry)
-        "#{status_entry[:tasklist].last rescue nil}#{status_entry[:text] rescue nil}"
+        self.status_log << info.merge(timestamp: ::Time.now).cleanup
       end
 
     end

data/lib/libis/workflow/worker.rb

@@ -11,12 +11,12 @@ module Libis
       include Sidekiq::Worker
 
       def perform(job_config, options = {})
-        job = self.class.configure(job_config, options)
+        job = configure(job_config, options)
         options[:interactive] = false
         job.execute options
       end
 
-      def self.configure(job_config, options = {})
+      def configure(job_config, options = {})
         log_path = options.delete :log_path
         if log_path
           Config.logger = ::Logger.new(
@@ -25,7 +25,7 @@ module Libis
               (options.delete(:log_shift_size) || 1024 ** 2)
           )
           Config.logger.formatter = ::Logger::Formatter.new
-          Config.logger.level = ::Logger::DEBUG
+          Config.logger.level = (options.delete(:log_level) || ::Logger::DEBUG)
         end
         get_job(job_config)
       end

data/lib/libis/workflow.rb

@@ -12,11 +12,14 @@ module Libis
       autoload :FileItem, 'libis/workflow/base/file_item'
       autoload :DirItem, 'libis/workflow/base/dir_item'
       autoload :Logger, 'libis/workflow/base/logger'
+      autoload :Logging, 'libis/workflow/base/logging'
       autoload :Job, 'libis/workflow/base/job'
       autoload :Run, 'libis/workflow/base/run'
       autoload :Workflow, 'libis/workflow/base/workflow'
     end
 
+    autoload :Status, 'libis/workflow/status'
+
     autoload :WorkItem, 'libis/workflow/work_item'
     autoload :FileItem, 'libis/workflow/file_item'
     autoload :DirItem, 'libis/workflow/dir_item'
@@ -25,6 +28,7 @@ module Libis
     autoload :Job, 'libis/workflow/job'
     autoload :Run, 'libis/workflow/run'
     autoload :Task, 'libis/workflow/task'
+    autoload :TaskGroup, 'libis/workflow/task_group'
 
     autoload :Worker, 'libis/workflow/worker'
 

data/spec/task_spec.rb

@@ -10,7 +10,6 @@ describe 'Task' do
 
     expect(task.parent).to eq nil
     expect(task.name).to eq 'Task'
-    expect(task.parameter(:abort_on_error)).to eq false
 
   end