libis-workflow 2.0.beta.3

data/lib/libis/workflow/config.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+
+require 'singleton'
+require 'logger'
+
+module Libis
+  module Workflow
+
+    class Config
+      include Singleton
+
+      attr_accessor :logger, :workdir, :taskdir, :itemdir
+
+      private
+
+      def initialize
+        Config.require_all(File.join(File.dirname(__FILE__), 'tasks'))
+        @logger = ::Logger.new STDOUT
+        set_formatter
+        @workdir = './work'
+        self.taskdir = './tasks'
+        self.itemdir = './items'
+      end
+
+      public
+
+      def set_formatter(formatter = nil)
+        @logger.formatter = formatter || proc do |severity, time, progname, msg|
+          "%s, [%s#%d] %5s -- %s: %s\n" % [severity[0..0],
+                                           (time.strftime('%Y-%m-%dT%H:%M:%S.') << '%06d ' % time.usec),
+                                           $$, severity, progname, msg]
+        end
+      end
+
+      def self.logger
+        instance.logger
+      end
+
+      def self.workdir
+        instance.workdir
+      end
+
+      def self.taskdir
+        instance.taskdir
+      end
+
+      def self.itemdir
+        instance.itemdir
+      end
+
+      def self.logger=(log)
+        instance.logger = log
+      end
+
+      def self.set_formatter(formatter = nil)
+        instance.set_formatter(formatter)
+      end
+
+      def self.workdir=(dir)
+        instance.workdir = dir
+      end
+
+      def taskdir=(dir)
+        @taskdir = dir
+        Config.require_all dir
+      end
+
+      def self.taskdir=(dir)
+        instance.taskdir = dir
+      end
+
+      def itemdir=(dir)
+        @itemdir = dir
+        Config.require_all dir
+      end
+
+      def self.itemdir=(dir)
+        instance.itemdir = dir
+      end
+
+      def self.require_all(dir)
+        return unless Dir.exist?(dir)
+        Dir.glob(File.join(dir, '*.rb')).each do |filename|
+          #noinspection RubyResolve
+          require filename
+        end
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/message_registry.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+require 'singleton'
+
+module Libis
+  module Workflow
+    class MessageRegistry
+      include Singleton
+
+      def initialize
+        @message_db = {}
+      end
+
+      def register_message(id, message)
+        @message_db[id] = message
+      end
+
+      def get_message(id)
+        @message_db[id]
+      end
+
+      def self.register_message(id, message)
+        self.instance.register_message id, message
+      end
+
+      def self.get_message(id)
+        self.instance.get_message id
+      end
+
+    end
+  end
+end

data/lib/libis/workflow/run.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+require 'libis/workflow/config'
+require 'libis/workflow/workflow'
+
+require 'libis/workflow/base/run'
+
+module Libis
+  module Workflow
+
+    class Run
+      include ::Libis::Workflow::Base::Run
+
+      attr_accessor :start_date, :tasks, :workflow
+
+      def initialize
+        @start_date = Time.now
+        @tasks = nil
+        @workflow = nil
+        # noinspection RubySuperCallWithoutSuperclassInspection
+        super
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/task.rb

@@ -0,0 +1,259 @@
+# encoding: utf-8
+require 'backports/rails/hash'
+require 'backports/rails/string'
+
+require 'libis/tools/parameter'
+
+require 'libis/workflow'
+require 'libis/workflow/base/logger'
+
+module Libis
+  module Workflow
+
+    # noinspection RubyTooManyMethodsInspection
+    class Task
+      include Base::Logger
+      include ::Libis::Tools::ParameterContainer
+
+      attr_accessor :parent, :name, :options, :workitem, :tasks
+
+      parameter abort_on_error: false, description: 'Stop all tasks when an error occurs.'
+      parameter allways_run: false, description: 'Run this task, even if the item failed a previous task.'
+      parameter subitems: false, description: 'Do not process the given item, but only the subitems.'
+      parameter recursive: false, description: 'Run the task on all subitems recursively.'
+
+      def self.task_classes
+        ObjectSpace.each_object(::Class).select {|klass| klass < self}
+      end
+
+      def initialize(parent, cfg = {})
+        self.parent = parent
+        self.tasks = []
+        configure cfg
+      end
+
+      def <<(task)
+        self.tasks << task
+      end
+
+      def run(item)
+
+        check_item_type WorkItem, item
+
+        return if item.failed? unless options[:allways_run]
+
+        if options[:subitems]
+            log_started item
+            run_subitems item
+            log_done(item) unless item.failed?
+        else
+          run_item(item)
+        end
+
+        item.save
+
+      end
+
+      def run_item(item)
+
+        begin
+
+          self.workitem = item
+
+          log_started item
+
+          pre_process item
+          process_item item
+          post_process item
+
+        rescue WorkflowError => e
+          error e.message
+          log_failed item
+
+        rescue WorkflowAbort => e
+          item.status = to_status :failed
+          raise e if parent
+
+        rescue ::Exception => e
+          fatal 'Exception occured: %s', e.message
+          debug e.backtrace.join("\n")
+          log_failed item
+        end
+
+        log_done item unless item.failed?
+
+      end
+
+      def names
+        (self.parent.names rescue Array.new).push(name).compact
+      end
+
+      def namepath; self.names.join('/'); end
+
+      def apply_options(opts)
+        o = opts[self.name] || opts[self.names.join('/')]
+
+        default_values.each do |name,_|
+          next unless o.key?(name)
+          parameter = get_parameter_definition name
+          self.options[name] = parameter.parse(o[name])
+        end if o and o.is_a? Hash
+
+        self.tasks.each do |task|
+          task.apply_options opts
+        end
+      end
+
+      protected
+
+      def log_started(item)
+        item.status = to_status :started
+        debug 'Started', item
+      end
+
+      def log_failed(item, message = nil)
+        warn (message || 'Failed'), item
+        item.status = to_status :failed
+      end
+
+      def log_done(item)
+        debug 'Completed', item
+        item.status = to_status :done
+      end
+
+      def process_item(item)
+        process item
+        run_subitems(item) if options[:recursive]
+        run_subtasks item
+      end
+
+      def process(item)
+        # needs implementation unless there are subtasks
+        raise RuntimeError, 'Should be overwritten' if self.tasks.empty?
+      end
+
+      def pre_process(_)
+        # optional implementation
+      end
+
+      def post_process(_)
+        # optional implementation
+      end
+
+      def get_root_item
+        self.workitem.root
+      end
+
+      def get_work_dir
+        get_root_item.work_dir
+      end
+
+      def capture_cmd(cmd, *opts)
+        out = StringIO.new
+        err = StringIO.new
+        $stdout = out
+        $stderr = err
+        status = system cmd, *opts
+        return [status, out.string, err.string]
+      ensure
+        $stdout = STDOUT
+        $stderr = STDERR
+      end
+
+      def run_subitems(parent_item)
+        items = subitems parent_item
+        failed = passed = 0
+        items.each_with_index do |item, i|
+          debug 'Processing subitem (%d/%d): %s', parent_item, i+1, items.count, item.to_s
+          run_item item
+          if item.failed?
+            failed += 1
+            if options[:abort_on_error]
+              error 'Aborting ...', parent_item
+              raise WorkflowAbort.new "Aborting: task #{name} failed on #{item}"
+            end
+          else
+            passed += 1
+          end
+        end
+        if failed > 0
+          warn '%d subitem(s) failed', parent_item, failed
+          if failed == items.count
+            error 'All subitems have failed', parent_item
+            log_failed parent_item
+            return
+          end
+        end
+        debug '%d of %d subitems passed', parent_item, passed, items.count if items.count > 0
+      end
+
+      def run_subtasks(item)
+        tasks = subtasks item
+        tasks.each_with_index do |task, i|
+          debug 'Running subtask (%d/%d): %s', item, i+1, tasks.count, task.name
+          task.run item
+          if item.failed?
+            if task.options[:abort_on_error]
+              error 'Aborting ...'
+              raise WorkflowAbort.new "Aborting: task #{task.name} failed on #{item}"
+            end
+            return
+          end
+        end
+      end
+
+      def configure(cfg)
+        self.name = cfg[:name] || (cfg[:class] || self.class).to_s.split('::').last
+        self.options =
+            default_values.merge(
+                cfg[:options] || {}
+            ).merge(
+                cfg.reject { |k, _| [:options].include? k.to_sym }
+            ).symbolize_keys!
+      end
+
+      def to_status(text)
+        [text.to_s.capitalize, self.names]
+      end
+
+      def check_item_type(klass, item = nil)
+        item ||= self.workitem
+        unless item.is_a? klass.to_s.constantize
+          raise WorkflowError, "Workitem is of wrong type : #{item.class} - expected #{klass.to_s}"
+        end
+      end
+
+      def item_type?(klass, item = nil)
+        item ||= self.workitem
+        item.is_a? klass.to_s.constantize
+      end
+
+      private
+
+      def subtasks(item = nil)
+        self.tasks.map do |task|
+          ((item || self.workitem).failed? and not task.options[:always_run]) ? nil : task
+        end.compact
+      end
+
+      def subitems(item = nil)
+        items = (item || workitem).items
+        return items if self.options[:always_run]
+        items.reject { |i| i.failed? }
+      end
+
+      def default_values
+        self.class.default_values
+      end
+
+      def self.default_values
+        parameters.inject({}) do |hash,parameter|
+          hash[parameter.first] = parameter.last[:default]
+          hash
+        end
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/tasks/analyzer.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+require 'libis/workflow/task'
+
+module Libis
+  module Workflow
+    module Tasks
+
+      class Analyzer < Task
+
+        def default_options
+          { quiet: true, allways_run: true }
+        end
+
+        def run(item)
+
+          item.properties[:ingest_failed] = item.failed?
+
+          item.log_history.each do |log|
+            level = log[:severity]
+            item.summary[level] ||= 0
+            item.summary[level] += 1
+          end
+
+          item.each do |i|
+            run i
+            i.summary.each do |level, count|
+              item.summary[level] ||= 0
+              item.summary[level] += (count || 0)
+            end
+          end
+
+          item.save
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/workflow/version.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+
+module Libis
+  module Workflow
+    VERSION = '2.0.beta.3' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
+  end
+end

data/lib/libis/workflow/worker.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require 'sidekiq'
+
+require 'libis/workflow/config'
+require 'libis/workflow/workflow'
+
+module Libis
+  module Workflow
+
+    class Worker
+      include Sidekiq::Worker
+
+      def perform(workflow_config, options = {})
+        workflow = self.class.configure(workflow_config, options)
+        options[:interactive] = false
+        workflow.run options
+      end
+
+      def self.configure(workflow_config, options = {})
+        log_path = options.delete :log_path
+        if log_path
+          Config.logger = ::Logger.new(
+              File.join(log_path, "#{workflow_config}.log"),
+              (options.delete(:log_shift_age) || 'daily'),
+              (options.delete(:log_shift_size) || 1024 ** 2)
+          )
+          Config.logger.formatter = ::Logger::Formatter.new
+          Config.logger.level = ::Logger::DEBUG
+        end
+        get_workflow(workflow_config)
+      end
+
+      def get_workflow(workflow_config)
+        workflow = ::Libis::Workflow::Workflow.new
+        workflow.configure workflow_config
+        workflow
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/workflow.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+require 'backports/rails/string'
+require 'backports/rails/hash'
+
+require 'libis/workflow/config'
+require 'libis/workflow/task'
+require 'libis/workflow/tasks/analyzer'
+
+require 'libis/workflow/base/workflow'
+
+module Libis
+  module Workflow
+
+    class Workflow
+      include ::Libis::Workflow::Base::Workflow
+
+      attr_accessor :name, :description, :config
+
+      def initialize
+        @name = ''
+        @descripition = ''
+        @config = Hash.new
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/workitems/dir_item.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+require 'libis/workflow/workitems/file_item'
+
+module Libis
+  module Workflow
+
+    module DirItem
+      include ::Libis::Workflow::FileItem
+    end
+  end
+end

data/lib/libis/workflow/workitems/file_item.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+
+require 'digest'
+
+require 'libis/workflow/workitems/work_item'
+
+module Libis
+  module Workflow
+
+    module FileItem
+      include WorkItem
+
+      def filename
+        File.basename(self.properties[:filename]) || self.properties[:link]
+      end
+
+      def name
+        self.properties[:name] || self.filename
+      end
+
+      def filelist
+        (self.parent.filelist rescue Array.new).push(filename).compact
+      end
+
+      def filepath
+        self.filelist.join('/')
+      end
+
+      def fullpath
+        self.properties[:filename]
+      end
+
+      def filename=(name)
+        begin
+          stats = ::File.stat name
+          self.properties[:size] = stats.size
+          self.properties[:access_time] = stats.atime
+          self.properties[:modification_time] = stats.mtime
+          self.properties[:creation_time] = stats.ctime
+          self.properties[:mode] = stats.mode
+          self.properties[:uid] = stats.uid
+          self.properties[:gid] = stats.gid
+          set_checksum(:MD5, ::Digest::MD5.hexdigest(File.read(name))) if File.file?(name)
+        rescue
+          # ignored
+        end
+        self.properties[:filename] = name
+      end
+
+      def checksum(checksum_type)
+        self.properties[('checksum_' + checksum_type.to_s.downcase).to_sym]
+      end
+
+      def set_checksum(checksum_type, value)
+        self.properties[('checksum_' + checksum_type.to_s.downcase).to_sym] = value
+      end
+
+      def link
+        self.properties[:link]
+      end
+
+      def link=(name)
+        self.properties[:link] = name
+      end
+
+      def set_info(info)
+        info.each do |k, v|
+          self.properties[k] = v
+        end
+      end
+
+      def safe_name
+        self.name.to_s.gsub(/[^\w.-]/) { |s| '%%%02x' % s.ord }
+      end
+
+    end
+  end
+end