libis-workflow 2.0.beta.19-java

data/lib/libis/workflow/job.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+
+require 'libis/workflow/base/job'
+require 'libis/workflow/workflow'
+require 'libis/workflow/run'
+
+module Libis
+  module Workflow
+
+    class Job
+      include ::Libis::Workflow::Base::Job
+
+      attr_accessor :name, :description, :workflow, :run_object, :input
+
+      def initialize
+        @name = ''
+        @description = ''
+        @input = Hash.new
+        @workflow = ::Libis::Workflow::Workflow.new
+        @run_object = ::Libis::Workflow::Run.new
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/message_registry.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+require 'singleton'
+
+module Libis
+  module Workflow
+    class MessageRegistry
+      include Singleton
+
+      def initialize
+        @message_db = {}
+      end
+
+      def register_message(id, message)
+        @message_db[id] = message
+      end
+
+      def get_message(id)
+        @message_db[id]
+      end
+
+      def self.register_message(id, message)
+        self.instance.register_message id, message
+      end
+
+      def self.get_message(id)
+        self.instance.get_message id
+      end
+
+    end
+  end
+end

data/lib/libis/workflow/run.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+
+require 'libis/workflow/config'
+require 'libis/workflow/workflow'
+
+require 'libis/workflow/base/run'
+require 'libis/workflow/work_item'
+
+module Libis
+  module Workflow
+
+    class Run < ::Libis::Workflow::WorkItem
+      include ::Libis::Workflow::Base::Run
+
+      attr_accessor :start_date, :job
+
+      def initialize
+        @start_date = Time.now
+        @job = nil
+        super
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/status.rb

@@ -0,0 +1,83 @@
+module Libis
+  module Workflow
+    module Status
+
+      STATUS = {
+          NOT_STARTED: 0,
+          STARTED: 1,
+          DONE: 2,
+          ASYNC_WAIT: 3,
+          ASYNC_HALT: 4,
+          FAILED: 5
+      }
+
+      # Changes the status of the object. The status changed is logged in the status_log with the current timestamp.
+      #
+      # @param [Array] x Array with status and task
+      def status=(x)
+        s, task = x
+        self.add_status_log(task: task, status: s)
+        self.save
+      end
+
+      # Get last known status symbol for a given task
+      #
+      # @param [String] task task name to check item status for
+      # @return [Symbol] the status code
+      def status(task = nil)
+        entry = status_entry(task)
+        status_symbol(entry[:status]) rescue :NOT_STARTED
+      end
+
+      # Gets the last known status label of the object.
+      #
+      # @param [String] task name of task to get the status for
+      # @return [String] status label ( = task name + status )
+      def status_label(task = nil)
+        entry = self.status_entry(task)
+        "#{entry[:task] rescue nil}#{entry[:status].capitalize rescue nil}"
+      end
+
+      # Check status of the object.
+      #
+      # @param [Symbol] state status to look for
+      # @param [String] task name of task whose status to check
+      # @return [Boolean] true if the object status matches
+      def check_status(state, task = nil)
+        self.status(task) == state
+      end
+
+      # Compare status with current status of the object.
+      #
+      # @param [Symbol] state
+      # @return [Integer] 1, 0 or -1 depnding on which
+      def compare_status(state, task = nil)
+        STATUS[self.status(task)] <=> STATUS[state]
+      end
+
+      protected
+
+      # Get last known status entry for a given task
+      #
+      # @param [String] task task name to check item status for
+      # @return [Hash] the status entry
+      def status_entry(task = nil)
+        return self.status_log.last if task.blank?
+        self.status_log.select { |entry| entry[:task] == task }.last
+      end
+
+      # Convert String, Symbol or Integer to correct symbol for the status.
+      # If the input value is nil, the fist status entry is returned.
+      #
+      # @param [String|Symbol|Integer] x string, symbol or integer for status code.
+      # @return [Symbol] the corresponding STATUS symbol
+      def status_symbol(x)
+        return STATUS.key(x) if x.is_a?(Integer)
+        return x if STATUS.has_key?(x)
+        x = x.to_s.upcase.to_sym
+        STATUS.has_key?(x) ? x : nil
+      end
+
+    end
+  end
+end

data/lib/libis/workflow/task.rb

@@ -0,0 +1,287 @@
+# encoding: utf-8
+require 'backports/rails/hash'
+require 'backports/rails/string'
+
+require 'libis/tools/parameter'
+require 'libis/tools/extend/hash'
+
+require 'libis/workflow'
+
+module Libis
+  module Workflow
+
+    # noinspection RubyTooManyMethodsInspection
+    class Task
+      include ::Libis::Workflow::Base::Logger
+      include ::Libis::Tools::ParameterContainer
+
+      attr_accessor :parent, :name, :workitem
+
+      parameter quiet: false, description: 'Prevemt generating log output.'
+      parameter recursive: false, description: 'Run the task on all subitems recursively.'
+      parameter retry_count: 0, description: 'Number of times to retry the task.'
+      parameter retry_interval: 10, description: 'Number of seconds to wait between retries.'
+
+      def self.task_classes
+        ObjectSpace.each_object(::Class).select { |klass| klass < self }
+      end
+
+      def initialize(parent, cfg = {})
+        @subitems_stopper = false
+        @subtasks_stopper = false
+        self.parent = parent
+        configure cfg
+      end
+
+      def <<(task)
+        raise Libis::WorkflowError, "Processing task '#{self.namepath}' is not allowed to have subtasks."
+      end
+
+      # @param [Libis::Workflow::Base::WorkItem] item
+      def run(item)
+        check_item_type ::Libis::Workflow::Base::WorkItem, item
+        self.workitem = item
+
+        case self.action
+          when :retry
+            return if item.check_status(:DONE, self.namepath)
+          when :failed
+            return
+          else
+        end
+
+        (parameter(:retry_count)+1).times do
+
+          run_item(item)
+
+          case item.status(self.namepath)
+            when :DONE
+              self.action = :run
+              return
+            when :ASYNC_WAIT
+              self.action = :retry
+            when :ASYNC_HALT
+              break
+            when :FAILED
+              break
+            else
+              return
+          end
+
+          self.action = :retry
+
+          sleep(parameter(:retry_interval))
+
+        end
+
+        item.get_run.action = :failed
+
+      rescue WorkflowError => e
+        error e.message, item
+        update_status item, :FAILED
+
+      rescue WorkflowAbort => e
+        update_status item, :FAILED
+        raise e if parent
+
+      rescue ::Exception => e
+        update_status item, :FAILED
+        fatal "Exception occured: #{e.message}", item
+        debug e.backtrace.join("\n")
+
+      ensure
+        item.save
+
+      end
+
+      def names
+        (self.parent.names rescue Array.new).push(name).compact
+      end
+
+      def namepath;
+        self.names.join('/');
+      end
+
+      def apply_options(opts)
+        o = {}
+        o.merge!(opts[self.class.to_s] || {})
+        o.merge!(opts[self.name] || opts[self.names.join('/')] || {})
+        o.key_strings_to_symbols! recursive: true
+
+        if o and o.is_a? Hash
+          default_values.each do |name, _|
+            next unless o.key?(name)
+            next if o[name].nil?
+            paramdef = get_parameter_definition name
+            value = paramdef.parse(o[name])
+            self.parameter(name, value)
+          end
+        end
+
+      end
+
+      protected
+
+      def configure(cfg)
+        self.name = cfg[:name] || (cfg[:class] || self.class).to_s.split('::').last
+        (cfg[:options] || {}).merge(
+            cfg.reject { |k, _| [:options, :name, :class].include? k.to_sym }
+        ).symbolize_keys.each do |k, v|
+          self.parameter(k, v)
+        end
+      end
+
+      def run_item(item)
+        @item_skipper = false
+
+        pre_process(item)
+
+        set_status item, :STARTED
+
+        self.process item unless @item_skipper
+
+        run_subitems(item) if parameter(:recursive)
+
+        update_status item, :DONE
+
+        post_process item
+      end
+
+      def pre_process(_)
+        true
+        # optional implementation
+      end
+
+      def post_process(_)
+        # optional implementation
+      end
+
+      def run_subitems(parent_item)
+        return unless check_processing_subitems
+        return unless parent_item.count > 0
+
+        status = Hash.new(0)
+        subitems(parent_item).each_with_index do |item, i|
+          debug 'Processing subitem (%d/%d): %s', parent_item, i+1, parent_item.count, item.to_s
+          run_item item
+          status[item.status(self.namepath)] += 1
+        end
+
+        debug '%d of %d subitems passed', parent_item, status[:DONE], parent_item.count
+        substatus_check(status, parent_item, 'item')
+      end
+
+      def substatus_check(status, item, task_or_item)
+        if (failed = status[:FAILED] > 0)
+          warn "%d sub#{task_or_item}(s) failed", item, failed
+          update_status(item, :FAILED)
+        end
+
+        if (halted = status[:ASYNC_HALT] > 0)
+          warn "%d sub#{task_or_item}(s) halted in async process", item, halted
+          update_status(item, :ASYNC_HALT)
+        end
+
+        if (waiting = status[:ASYNC_WAIT] > 0)
+          warn "waiting for %d sub#{task_or_item}(s) in async process", item, waiting
+          update_status(item, :ASYNC_WAIT)
+        end
+
+        update_status(item, :DONE)
+      end
+
+      def capture_cmd(cmd, *opts)
+        out = StringIO.new
+        err = StringIO.new
+        $stdout = out
+        $stderr = err
+        status = system cmd, *opts
+        return [status, out.string, err.string]
+      ensure
+        $stdout = STDOUT
+        $stderr = STDERR
+      end
+
+      def action=(action)
+        self.workitem.get_run.action = action
+      end
+
+      def action
+        self.workitem.get_run.action
+      end
+
+      def get_root_item(item = nil)
+        (item || self.workitem).get_root
+      end
+
+      def get_work_dir(item = nil)
+        get_root_item(item).work_dir
+      end
+
+      def stop_processing_subitems
+        @subitems_stopper = true if parameter(:recursive)
+      end
+
+      def check_processing_subitems
+        if @subitems_stopper
+          @subitems_stopper = false
+          return false
+        end
+        true
+      end
+
+      def skip_processing_item
+        @item_skipper = true
+      end
+
+      def update_status(item, state)
+        return nil unless item.compare_status(state, self.namepath) < 0
+        set_status(item, state)
+      end
+
+      def set_status(item, state)
+        item.status = to_status(state)
+        state
+      end
+
+      def to_status(state)
+        [state, self.namepath]
+      end
+
+      def check_item_type(klass, item = nil)
+        item ||= self.workitem
+        unless item.is_a? klass.to_s.constantize
+          raise WorkflowError, "Workitem is of wrong type : #{item.class} - expected #{klass.to_s}"
+        end
+      end
+
+      def item_type?(klass, item = nil)
+        item ||= self.workitem
+        item.is_a? klass.to_s.constantize
+      end
+
+      private
+
+      def subtasks
+        self.tasks
+      end
+
+      def subitems(item = nil)
+        (item || self.workitem).get_items
+      end
+
+      def default_values
+        self.class.default_values
+      end
+
+      def self.default_values
+        parameter_defs.inject({}) do |hash, parameter|
+          hash[parameter.first] = parameter.last[:default]
+          hash
+        end
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/task_group.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require_relative 'task'
+
+module Libis
+  module Workflow
+
+    # noinspection RubyTooManyMethodsInspection
+    class TaskGroup < Libis::Workflow::Task
+
+      attr_accessor :tasks
+
+      def initialize(parent, cfg = {})
+        self.tasks = []
+        super parent, cfg
+      end
+
+      def <<(task)
+        self.tasks << task
+      end
+
+      def apply_options(opts)
+        super opts
+        self.tasks.each do |task|
+          task.apply_options opts
+        end
+      end
+
+      protected
+
+      def process(item)
+
+        return unless check_processing_subtasks
+
+        tasks = subtasks
+        return unless tasks.count > 0
+
+        status = Hash.new(0)
+        tasks.each_with_index do |task, i|
+          debug 'Running subtask (%d/%d): %s', item, i+1, tasks.count, task.name
+          task.run item
+          status[item.status(task.namepath)] += 1
+        end
+
+        substatus_check(status, item, 'task')
+      end
+
+      def stop_processing_subtasks
+        @subtasks_stopper= true
+      end
+
+      def check_processing_subtasks
+        if @subtasks_stopper
+          @subtasks_stopper = false
+          return false
+        end
+        true
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/tasks/analyzer.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+require 'libis/workflow/task'
+
+module Libis
+  module Workflow
+    module Tasks
+
+      class Analyzer < Task
+
+        parameter quiet: true, frozen: true
+        parameter recursive: false, frozen: true
+
+        # @param [Libis::Workflow::Base::WorkItem] item
+        def run(item)
+
+          item.properties[:ingest_failed] = item.check_status(:FAILED)
+
+          item.summary = {}
+          item.log_history.each do |log|
+            level = log[:severity]
+            item.summary[level] ||= 0
+            item.summary[level] += 1
+          end
+
+          item.each do |i|
+            run i
+            i.summary.each do |level, count|
+              item.summary[level] ||= 0
+              item.summary[level] += (count || 0)
+            end
+          end
+
+        rescue RuntimeError => ex
+
+          puts 'Failed to analyze item: %s - %s' % [item.class, item.name]
+          puts 'Exception: %s' % ex.message
+
+        ensure
+
+          item.save
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/workflow/version.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+
+module Libis
+  module Workflow
+    VERSION = '2.0.beta.19' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
+  end
+end

data/lib/libis/workflow/work_item.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+require 'libis/tools/extend/hash'
+require 'libis/workflow/base/work_item'
+
+module Libis
+  module Workflow
+
+    # In-memory implementation of ::Libis::Workflow::Base::WorkItem
+    class WorkItem
+      include ::Libis::Workflow::Base::WorkItem
+
+      attr_accessor :parent
+      attr_accessor :items
+      attr_accessor :options, :properties
+      attr_accessor :log_history, :status_log
+      attr_accessor :summary
+
+      def initialize
+        self.parent = nil
+        self.items = []
+        self.options = {}
+        self.properties = {}
+        self.log_history = []
+        self.status_log = []
+        self.summary = {}
+      end
+
+      protected
+
+      def add_log_entry(msg)
+        # noinspection RubyResolve
+        self.log_history << msg.merge(c_at: ::Time.now)
+      end
+
+      def add_status_log(info)
+        # noinspection RubyResolve
+        self.status_log << info.merge(timestamp: ::Time.now).cleanup
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/worker.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require 'sidekiq'
+
+require 'libis/workflow/config'
+require 'libis/workflow/workflow'
+
+module Libis
+  module Workflow
+
+    class Worker
+      include Sidekiq::Worker
+
+      def perform(job_config, options = {})
+        job = configure(job_config, options)
+        options[:interactive] = false
+        job.execute options
+      end
+
+      def configure(job_config, options = {})
+        log_path = options.delete :log_path
+        if log_path
+          Config.logger = ::Logger.new(
+              File.join(log_path, "#{job_config[:name]}.log"),
+              (options.delete(:log_shift_age) || 'daily'),
+              (options.delete(:log_shift_size) || 1024 ** 2)
+          )
+          Config.logger.formatter = ::Logger::Formatter.new
+          Config.logger.level = (options.delete(:log_level) || ::Logger::DEBUG)
+        end
+        get_job(job_config)
+      end
+
+      def get_job(job_config)
+        job = ::Libis::Workflow::Job.new
+        job.configure job_config
+        job
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/workflow.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+require 'libis/workflow/base/workflow'
+
+module Libis
+  module Workflow
+
+    class Workflow
+      include ::Libis::Workflow::Base::Workflow
+
+      attr_accessor :name, :description, :config
+
+      def initialize
+        @name = ''
+        @description = ''
+        @config = Hash.new
+      end
+
+    end
+
+  end
+end