libis-workflow 2.0.beta.19-java

data/lib/libis/workflow/base/job.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+
+require 'libis/tools/parameter'
+
+module Libis
+  module Workflow
+    module Base
+
+      # This is the base module for Jobs.
+      #
+      # This module lacks the implementation for the data attributes. It functions as an interface that describes the
+      # common functionality regardless of the storage implementation. These attributes require some implementation:
+      #
+      # - name: [String] the name of the Job. The name will be used to identify the job. Each time a job is executed,
+      #     a Run will be created for the associated workflow. The Run will get a name that starts with the job name
+      #     and ends with the date and time the Run was first started. As such this name attribute serves as an
+      #     identifier and should be treated as such. If possible it should be unique.
+      # - description: [String] optional information about the job.
+      # - workflow: [Object] the workflow containing the tasks that need to run.
+      # - run_obj: [String] the full class name of the Run implementation object that should be created when the
+      #     Job is executed.
+      # - input: [Hash] workflow input parameter values. Each input parameter of the workflow can be set by the entries
+      #     in this Hash.
+      #
+      # A minimal in-memory implementation could be:
+      #
+      # class Job
+      #   include ::Libis::Workflow::Base::Job
+      #
+      #   attr_accessor :name, :description, :workflow, :run_object, :input
+      #
+      #   def initialize
+      #     @name = ''
+      #     @description = ''
+      #     @input = Hash.new
+      #     @workflow = ::Libis::Workflow::Workflow.new
+      #     @run_object = ::Libis::Workflow::Run.new
+      #   end
+      #
+      # end
+      #
+      module Job
+
+        def run_name(timestamp = Time.now)
+          "#{self.name}-#{timestamp.strftime('%Y%m%d%H%M%S')}"
+        end
+
+        def configure(cfg = {})
+          self.name ||= ''
+          self.description ||= ''
+          self.input ||= {}
+
+          self.name = cfg[:name] if cfg.has_key?(:name)
+          self.description = cfg[:description] if cfg.has_key?(:description)
+          self.workflow = cfg[:workflow] if cfg.has_key?(:workflow)
+          self.run_object = cfg[:run_object] if cfg.has_key?(:run_object)
+          self.input.merge!(cfg[:input] || {})
+        end
+
+        # noinspection RubyResolve
+        # @param [Hash] opts optional extra conguration values for this particular run
+        def execute(opts = {})
+          run = self.create_run_object
+          raise RuntimeError.new "Could not create instance of run object '#{self.run_object}'" unless run
+
+          run.job = self
+          run.options = self.input.merge(opts)
+          run.save
+
+          run.run
+
+          run
+        end
+
+        protected
+
+        # noinspection RubyResolve
+        def create_run_object
+          self.run_object.constantize.new
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/workflow/base/logger.rb

@@ -0,0 +1,30 @@
+require 'libis/tools/logger'
+
+module Libis
+  module Workflow
+    module Base
+      module Logger
+        include ::Libis::Tools::Logger
+
+        def message(severity, msg, *args)
+          item = self.workitem
+          item = args.shift if args.size > 0 and args[0].is_a?(WorkItem)
+
+          item.log_message(severity, to_msg(msg), *args) if item
+        end
+
+        def to_msg(msg)
+          case msg
+            when String
+              {text: msg}
+            when Integer
+              {id: msg}
+            else
+              {text: (msg.to_s rescue '')}
+          end.merge task: self.namepath
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/workflow/base/logging.rb

@@ -0,0 +1,76 @@
+module Libis
+  module Workflow
+    module Base
+      module Logging
+
+        # Helper function for the Tasks to add a log entry to the log_history.
+        #
+        # The supplied message structure is expected to contain the following fields:
+        # - :severity : ::Logger::Severity value
+        # - :id : optional message id
+        # - :text : message text
+        # - :task : list of tasks names (task hierarchy) that submits the message
+        #
+        # @param [Hash] message
+        def add_log(message = {})
+          msg = message_struct(message)
+          add_log_entry(msg)
+          self.save
+        end
+
+        def <=(message = {})
+          self.add_log(message)
+        end
+
+        # Add a structured message to the log history. The message text can be submitted as an integer or text. If an
+        # integer is submitted, it will be used to look up the text in the MessageRegistry. The message text will be
+        # passed to the % operator with the args parameter. If that failes (e.g. because the format string is not correct)
+        # the args value is appended to the message.
+        #
+        # @param [Symbol] severity
+        # @param [Hash] msg should contain message text as :id or :text and the hierarchical name of the task as :task
+        # @param [Array] args string format values
+        def log_message(severity, msg, *args)
+          # Prepare info from msg struct for use with string substitution
+          message_id, message_text = if msg[:id]
+                                       [msg[:id], MessageRegistry.instance.get_message(msg[:id])]
+                                     elsif msg[:text]
+                                       [0, msg[:text]]
+                                     else
+                                       [0, '']
+                                     end
+          task = msg[:task] || '*UNKNOWN*'
+          message_text = (message_text % args rescue "#{message_text} - #{args}")
+
+          self.add_log severity: severity, id: message_id.to_i, text: message_text, task: task
+          name = ''
+          begin
+            name = self.to_s
+            name = self.name
+            name = self.namepath
+          rescue
+            # do nothing
+          end
+          Config.logger.add(severity, message_text, ('%s - %s ' % [task, name]))
+        end
+
+        protected
+
+        SEV_LABEL = %w(DEBUG INFO WARN ERROR FATAL ANY) unless const_defined? :SEV_LABEL
+
+        # create and return a proper message structure
+        # @param [Hash] opts
+        def message_struct(opts = {})
+          opts.reverse_merge!(severity: ::Logger::INFO, code: nil, text: '')
+          {
+              severity: SEV_LABEL[opts[:severity]],
+              task: opts[:task],
+              code: opts[:code],
+              message: opts[:text]
+          }.cleanup
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/workflow/base/run.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+
+require 'fileutils'
+
+require 'libis/workflow/base/work_item'
+
+module Libis
+  module Workflow
+    module Base
+
+      # Base module for all workflow runs. It is created by an associated workflow when the workflow is executed.
+      #
+      # This module lacks the implementation for the data attributes. It functions as an interface that describes the
+      # common functionality regardless of the storage implementation. These attributes require some implementation:
+      #
+      # - start_date: [Time] the timestamp of the execution of the run
+      # - job: [Object] a reference to the Job this Run belongs to
+      #
+      # Note that ::Libis::Workflow::Base::WorkItem is a parent module and therefore requires implementation of the
+      # attributes of that module too.
+      #
+      # A simple in-memory implementation can be found in ::Libis::Workflow::Run
+      module Run
+        include ::Libis::Workflow::Base::WorkItem
+
+        attr_accessor :tasks, :action
+
+        def work_dir
+          # noinspection RubyResolve
+          dir = File.join(Config.workdir, self.name)
+          FileUtils.mkpath dir unless Dir.exist?(dir)
+          dir
+        end
+
+        def name
+          self.job.run_name(self.start_date)
+        end
+
+        def names
+          Array.new
+        end
+
+        def namepath
+          self.name
+        end
+
+        def workflow
+          self.job.workflow
+        end
+
+        # Execute the workflow.
+        #
+        # The action parameter defines how the execution of the tasks will behave:
+        #  - With the default :run action each task will be executed regardsless how the task performed on the item
+        #    previously.
+        #  - When using the :retry action a task will not perform on an item if it was successful the last time. This
+        #    allows you to retry a run when an temporary error (e.g. asynchronous wait or halt) occured.
+        #
+        # @param [Symbol] action the type of action to take during this run. :run or :retry
+        def run(action = :run)
+          self.action = action
+
+          self.start_date = Time.now
+
+          self.options = workflow.prepare_input(self.options)
+
+          self.tasks = workflow.tasks(self)
+          configure_tasks self.options
+
+
+          self.tasks.each do |task|
+            task.run self
+          end
+
+        end
+
+        protected
+
+        def configure_tasks(opts)
+          self.tasks.each { |task| task.apply_options opts }
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/workflow/base/work_item.rb

@@ -0,0 +1,176 @@
+# encoding: utf-8
+
+require 'backports/rails/hash'
+require 'libis/tools/extend/hash'
+
+require 'libis/workflow/config'
+require 'libis/workflow/status'
+require_relative 'logging'
+
+module Libis
+  module Workflow
+    module Base
+
+      # Base module for all work items.
+      #
+      # This module lacks the implementation for the data attributes. It functions as an interface that describes the
+      # common functionality regardless of the storage implementation. These attributes require some implementation:
+      #
+      # - status: [Symbol] the status field. Each task sets the status of the items it works on. Before starting processing
+      #     the status is set to "#{task_name}Started". After successfull processing it is set to "#{task_name}Done" and if
+      #     the task failed, it is set to "#{task_name}Failed". The status field can be used to perform real-time
+      #     monitoring, reporting and error-recovery or restart of the ingest.
+      #     The initial value for this attribute is :START.
+      # - parent: [Object|nil] a link to a parent work item. Work items can be organized in any hierarchy you think is
+      #     relevant for your workflow (e.g. directory[/directory...]/file/line or library/section/book/page). Of course
+      #     hierarchies are not mandatory.
+      # - items: [Array] a list of child work items. see above.
+      # - options: [Hash] a set of options for the task chain on how to deal with this work item. This attribute can be
+      #     used to fine-tune the behaviour of tasks for a particular work item.
+      # - properties: [Hash] a set of properties, typically collected during the workflow processing and used to store
+      #     final or intermediate resulst of tasks. The ::Lias::Ingester::FileItem module uses this attribute to store the
+      #     properties (e.g. size, checksum, ...) of the file it represents.
+      # - log_history: [Array] a list of all logging messages collected for this work item. Whenever a task logs a message
+      #     it will automatically be registered for the work item that it is processing or for the work item that was
+      #     supplied as the first argument.
+      # - status_log: [Array] a list of all status changes the work item went through.
+      # - summary: [Hash] collected statistics about the ingest for the work item and its children. This structure will
+      #     be filled in by the included task ::Lias::Ingester::Tasks::Analyzer wich is appended to the workflow by default.
+      #
+      # The module is created so that it is possible to implement an ActiveRecord/Datamapper/... implementation easily.
+      # A simple in-memory implementation would require:
+      #
+      # attr_accessor :parent
+      # attr_accessor :items
+      # attr_accessor :options, :properties
+      # attr_accessor :log_history, :status_log
+      # attr_accessor :summary
+      #
+      # def initialize
+      #   self.parent = nil
+      #   self.items = []
+      #   self.options = {}
+      #   self.properties = {}
+      #   self.log_history = []
+      #   self.status_log = []
+      #   self.summary = {}
+      # end
+      #
+      # protected
+      #
+      # ## Methods below should be adapted to match the implementation of the log arrays
+      #
+      # def add_log_entry(msg)
+      #   self.log_history << msg.merge(c_at: ::Time.now)
+      # end
+      #
+      # def add_status_log(message, tasklist = nil)
+      #   self.status_log << { c_at: ::Time.now, tasklist: tasklist, text: message }.cleanup
+      # end
+      #
+      #
+      module WorkItem
+        include Enumerable
+        include Libis::Workflow::Status
+        include Libis::Workflow::Base::Logging
+
+        # String representation of the identity of the work item.
+        #
+        # You may want to overwrite this method as it tries the :name property or whatever #inspect returns if that
+        # failes. Typically this should return the key value, file name or id number. If that's what your :name property
+        # contains, you're fine.
+        #
+        # @return [String] string identification for this work item.
+        def name
+          # noinspection RubyResolve
+          self.properties[:name] || self.inspect
+        end
+
+        def to_s;
+          self.name;
+        end
+
+        def names
+          (self.parent.names rescue Array.new).push(name).compact
+        end
+
+        def namepath;
+          self.names.join('/');
+        end
+
+        # File name safe version of the to_s output.
+        #
+        # The output should be safe to use as a file name to store work item
+        # data. Typical use is when extra file items are created by a task and need to be stored on disk. The default
+        # implementation URL-encodes (%xx) all characters except alphanumeric, '.' and '-'.
+        #
+        # @return [String] file name
+        def to_filename
+          self.to_s.gsub(/[^\w.-]/) { |s| '%%%02x' % s.ord }
+        end
+
+        # Iterates over the work item clients and invokes code on each of them.
+        def each
+          self.items.each { |item| yield item }
+        end
+
+        # Add a child work item
+        #
+        # @param [WorkItem] item to be added to the child list :items
+        def add_item(item)
+          return self unless item and item.is_a? Libis::Workflow::Base::WorkItem
+          self.items << item
+          item.parent = self
+          self.save!
+          item.save!
+          self
+        end
+
+        alias_method :<<, :add_item
+
+        def get_items
+          self.items.dup
+        end
+
+        def item_count
+          self.items.size
+        end
+
+        # Return item's parent
+        # @return [Libis::Workflow::Base::WorkItem]
+        def get_parent
+          self.parent
+        end
+
+        # go up the hierarchy and return the topmost work item
+        #
+        # @return [Libis::Workflow::Base::WorkItem]
+        def get_root
+          self.get_parent && self.get_parent.is_a?(Libis::Workflow::Base::WorkItem) && self.get_parent.get_root || self
+        end
+
+        # Get the top
+        #
+        # @return [Libis::Workflow::Base::Run]
+        def get_run
+          return self if self.is_a?(Libis::Workflow::Base::Run)
+          self.get_parent && self.get_parent.get_run || nil
+        end
+
+        # Dummy method. It is a placeholder for DB backed implementations. Wherever appropriate WorkItem#save will be
+        # called to save the current item's state. If state needs to persisted, you should override this method or make
+        # sure your persistence layer implements it in your class.
+        def save
+        end
+
+        # Dummy method. It is a placeholder for DB backed implementations. Wherever appropriate WorkItem#save will be
+        # called to save the current item's state. If state needs to persisted, you should override this method or make
+        # sure your persistence layer implements it in your class.
+        def save!
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/workflow/base/workflow.rb

@@ -0,0 +1,153 @@
+# encoding: utf-8
+
+require 'libis/tools/parameter'
+require 'libis/workflow/task_group'
+
+module Libis
+  module Workflow
+    module Base
+
+      # This is the base module for Workflows.
+      #
+      # This module lacks the implementation for the data attributes. It functions as an interface that describes the
+      # common functionality regardless of the storage implementation. These attributes require some implementation:
+      #
+      # - name: [String] the name of the Workflow. The name will be used to identify the workflow. Each time a workflow
+      #     is executed, a Run will be created. The Run will get a name that starts with the workflow name and ends with
+      #     the date and time the Run was started. As such this name attribute serves as an identifier and should be
+      #     treated as such. If possible is should be unique.
+      # - description: [String] more information about the workflow.
+      # - config: [Hash] detailed configuration for the workflow. The application assumes it behaves as a Hash and will
+      #     access it with [], merge! and delete methods. If your implementation decides to implement it with another
+      #     object, it should implement above methods. The config Hash requires the following keys:
+      #   - input: [Hash] all input parameter definitions where the key is the parameter name and the value is another
+      #       Hash with arguments for the parameter definition. It typically contains the following arguments:
+      #       - default: default value if no value specified when the workflow is executed
+      #       - propagate_to: the task name (or path) and parameter name that any set value for this parameter will be
+      #           propagated to. The syntax is <task name|task path>[#<parameter name>]. It the #<parameter name> part
+      #           is not present, the same name as the input parameter is used. If you want to push the value to
+      #           multiple task parameters, you can either supply an array of propagate paths or put them in a string
+      #           separated by a ','.
+      #   - tasks: [Array] task definitions that define the order in which the tasks should be executed for the workflow.
+      #       A task definition is a Hash with the following values:
+      #       - class: [String] the class name of the task including the module names
+      #       - name: [String] optional if class is present. A friendly name for the task that will be used in the logs.
+      #       - tasks: [Array] a list of subtask defintions for this task.
+      #
+      #       Additionally the task definition Hash may specify values for any other parameter that the task knows of.
+      #       All tasks have parameters 'quiet', 'recursive', 'retry_count' and 'retry_interval'. For more
+      #       information about these see the documentation of the task class.
+      #
+      #       A task definition does not require to have a 'class' entry. If not present the default
+      #       ::Libis::Workflow::TaskGroup class will be instatiated. It will do nothing itself, but will execute the
+      #       subtasks on the item(s). In such case a 'name' is mandatory.
+      #
+      # These values should be set by calling the #configure method which takes a Hash as argument with :name,
+      # :description, :input and :tasks keys.
+      #
+      # A minimal in-memory implementation could be:
+      #
+      # class Workflow
+      #   include ::Libis::Workflow::Base::Workflow
+      #
+      #   attr_accessor :name, :description, :config
+      #
+      #   def initialize
+      #     @name = ''
+      #     @description = ''
+      #     @config = Hash.new
+      #   end
+      #
+      # end
+      #
+      module Workflow
+
+        module ClassMethods
+          def require_all
+            Config.require_all(File.join(File.dirname(__FILE__), '..', 'tasks'))
+            # noinspection RubyResolve
+            Config.require_all(Config.taskdir)
+            # noinspection RubyResolve
+            Config.require_all(Config.itemdir)
+          end
+        end
+
+        def self.included(base)
+
+          base.extend ClassMethods
+        end
+
+        def configure(cfg)
+          self.name = cfg.delete(:name) || self.class.name
+          self.description = cfg.delete(:description) || ''
+          self.config.merge! input: {}, tasks: []
+          self.config.merge! cfg
+
+          self.class.require_all
+
+          unless self.config[:tasks].last[:class] && self.config[:tasks].last[:class].split('::').last == 'Analyzer'
+            self.config[:tasks] << {class: '::Libis::Workflow::Tasks::Analyzer'}
+          end
+
+          self.config
+        end
+
+        def input
+          self.config[:input].inject({}) do |hash, input_def|
+            name = input_def.first.to_sym
+            default = input_def.last[:default]
+            parameter = ::Libis::Tools::Parameter.new name, default
+            input_def.last.each { |k, v| parameter[k.to_sym] = v }
+            hash[name] = parameter
+            hash
+          end
+        rescue
+          {}
+        end
+
+        # @param [Hash] options
+        def prepare_input(options)
+          result = {}
+          self.input.each do |key, parameter|
+            if options.has_key?(key)
+              value = parameter.parse(options[key])
+            elsif !parameter[:default].nil?
+              value = parameter[:default]
+            else
+              next
+            end
+            propagate_to = []
+            propagate_to = parameter[:propagate_to] if parameter[:propagate_to].is_a? Array
+            propagate_to = parameter[:propagate_to].split(/[\s,;]+/) if parameter[:propagate_to].is_a? String
+            result[key] = value if propagate_to.empty?
+            propagate_to.each do |target|
+              task_name, param_name = target.split('#')
+              param_name ||= key
+              result[task_name] ||= {}
+              result[task_name][param_name.to_sym] = value
+            end
+          end
+          result
+        end
+
+        def tasks(parent = nil)
+          self.config[:tasks].map do |cfg|
+            instantize_task(parent || self, cfg)
+          end
+        end
+
+        def instantize_task(parent, cfg)
+          task_class = Libis::Workflow::TaskGroup
+          task_class = cfg[:class].constantize if cfg[:class]
+          # noinspection RubyArgCount
+          task_instance = task_class.new(parent, cfg)
+          cfg[:tasks] && cfg[:tasks].map do |task_cfg|
+            task_instance << instantize_task(task_instance, task_cfg)
+          end
+          task_instance
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/workflow/base.rb

@@ -0,0 +1,7 @@
+require_relative 'base/logger'
+require_relative 'base/logging'
+require_relative 'base/work_item'
+require_relative 'base/file_item'
+require_relative 'base/dir_item'
+require_relative 'base/run'
+require_relative 'base/workflow'

data/lib/libis/workflow/config.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+require 'libis/tools/config'
+
+module Libis
+  module Workflow
+
+    # noinspection RubyConstantNamingConvention
+    Config = ::Libis::Tools::Config
+
+    Config.define_singleton_method(:require_all) do |dir|
+      Dir.glob(File.join(dir, '*.rb')).each do |filename|
+        # noinspection RubyResolve
+        require filename
+      end
+    end
+
+    Config.require_all(File.join(File.dirname(__FILE__), 'tasks'))
+    Config[:workdir] = './work'
+    Config[:taskdir] = './tasks'
+    Config[:itemdir] = './items'
+
+  end
+end

data/lib/libis/workflow/dir_item.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+require 'libis/workflow/file_item'
+
+module Libis
+  module Workflow
+
+    class DirItem < ::Libis::Workflow::FileItem
+    end
+
+  end
+end

data/lib/libis/workflow/file_item.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+
+require 'digest'
+
+require 'libis/workflow/base/file_item'
+require 'libis/workflow/work_item'
+
+module Libis
+  module Workflow
+
+    # noinspection RubyResolve
+    class FileItem < ::Libis::Workflow::WorkItem
+      include ::Libis::Workflow::Base::FileItem
+
+    end
+  end
+end