libis-workflow 2.0.beta.9 → 2.0.beta.10

data/lib/libis/workflow/base/work_item.rb

@@ -0,0 +1,246 @@
+# encoding: utf-8
+
+require 'backports/rails/hash'
+require 'libis/tools/extend/hash'
+
+require 'libis/workflow/config'
+
+module Libis
+  module Workflow
+    module Base
+
+      # Base module for all work items.
+      #
+      # This module lacks the implementation for the data attributes. It functions as an interface that describes the
+      # common functionality regardless of the storage implementation. These attributes require some implementation:
+      #
+      # - status: [Symbol] the status field. Each task sets the status of the items it works on. Before starting processing
+      #     the status is set to "#{task_name}Started". After successfull processing it is set to "#{task_name}Done" and if
+      #     the task failed, it is set to "#{task_name}Failed". The status field can be used to perform real-time
+      #     monitoring, reporting and error-recovery or restart of the ingest.
+      #     The initial value for this attribute is :START.
+      # - parent: [Object|nil] a link to a parent work item. Work items can be organized in any hierarchy you think is
+      #     relevant for your workflow (e.g. directory[/directory...]/file/line or library/section/book/page). Of course
+      #     hierarchies are not mandatory.
+      # - items: [Array] a list of child work items. see above.
+      # - options: [Hash] a set of options for the task chain on how to deal with this work item. This attribute can be
+      #     used to fine-tune the behaviour of tasks for a particular work item.
+      # - properties: [Hash] a set of properties, typically collected during the workflow processing and used to store
+      #     final or intermediate resulst of tasks. The ::Lias::Ingester::FileItem module uses this attribute to store the
+      #     properties (e.g. size, checksum, ...) of the file it represents.
+      # - log_history: [Array] a list of all logging messages collected for this work item. Whenever a task logs a message
+      #     it will automatically be registered for the work item that it is processing or for the work item that was
+      #     supplied as the first argument.
+      # - status_log: [Array] a list of all status changes the work item went through.
+      # - summary: [Hash] collected statistics about the ingest for the work item and its children. This structure will
+      #     be filled in by the included task ::Lias::Ingester::Tasks::Analyzer wich is appended to the workflow by default.
+      #
+      # The module is created so that it is possible to implement an ActiveRecord/Datamapper/... implementation easily.
+      # A simple in-memory implementation would require:
+      #
+      # attr_accessor :parent
+      # attr_accessor :items
+      # attr_accessor :options, :properties
+      # attr_accessor :log_history, :status_log
+      # attr_accessor :summary
+      #
+      # def initialize
+      #   self.parent = nil
+      #   self.items = []
+      #   self.options = {}
+      #   self.properties = {}
+      #   self.log_history = []
+      #   self.status_log = []
+      #   self.summary = {}
+      # end
+      #
+      # protected
+      #
+      # ## Methods below should be adapted to match the implementation of the log arrays
+      #
+      # def add_log_entry(msg)
+      #   self.log_history << msg.merge(c_at: ::Time.now)
+      # end
+      #
+      # def add_status_log(message, tasklist = nil)
+      #   self.status_log << { c_at: ::Time.now, tasklist: tasklist, text: message }.cleanup
+      # end
+      #
+      # def status_label(status_entry)
+      #   "#{status_entry[:tasklist].last rescue nil}#{status_entry[:text] rescue nil}"
+      # end
+      #
+      module WorkItem
+        include Enumerable
+
+        # String representation of the identity of the work item.
+        #
+        # You may want to overwrite this method as it tries the :name property or whatever #inspect returns if that
+        # failes. Typically this should return the key value, file name or id number. If that's what your :name property
+        # contains, you're fine.
+        #
+        # @return [String] string identification for this work item.
+        def name
+          # noinspection RubyResolve
+          self.properties[:name] || self.inspect
+        end
+
+        def to_s;
+          self.name;
+        end
+
+        def names
+          (self.parent.names rescue Array.new).push(name).compact
+        end
+
+        def namepath;
+          self.names.join('/');
+        end
+
+        # File name save version of the to_s output. The output should be safe to use as a file name to store work item
+        # data. Typical use is when extra file items are created by a task and need to be stored on disk. The default
+        # implementation URL-encodes (%xx) all characters except alphanumeric, '.' and '-'.
+        #
+        # @return [String] file name
+        def to_filename
+          self.to_s.gsub(/[^\w.-]/) { |s| '%%%02x' % s.ord }
+        end
+
+        # Gets the current status of the object.
+        #
+        # @return [Symbol] status code
+        def status
+          s = self.status_log.last
+          label = status_label(s)
+          label.empty? ? :NOT_STARTED : label.to_sym
+        end
+
+        # Changes the status of the object. The status changed is logged in the status_log with the current timestamp.
+        #
+        # @param [Symbol] s
+        def status=(s, tasklist = nil)
+          s, tasklist = s if s.is_a? Array
+          s = s.to_sym
+          add_status_log(s, tasklist)
+          self.save
+        end
+
+        # Check ingest status of the object. The status is checked to see if it ends in 'Failed'.
+        #
+        # @return [Boolean] true if the object failed, false otherwise
+        def failed?
+          self.status.to_s =~ /Failed$/i ? true : false
+        end
+
+        # Helper function for the Tasks to add a log entry to the log_history.
+        #
+        # The supplied message structure is expected to contain the following fields:
+        # - :severity : ::Logger::Severity value
+        # - :id : optional message id
+        # - :text : message text
+        # - :task : list of tasks names (task hierarchy) that submits the message
+        #
+        # @param [Hash] message
+        def add_log(message = {})
+          msg = message_struct(message)
+          add_log_entry(msg)
+          self.save
+        end
+
+        def <=(message = {})
+          ; self.add_log(message);
+        end
+
+        # Iterates over the work item clients and invokes code on each of them.
+        def each
+          self.items.each { |item| yield item }
+        end
+
+        # Add a child work item
+        #
+        # @param [WorkItem] item to be added to the child list :items
+        def add_item(item)
+          return self unless item and item.is_a? WorkItem
+          self.items << item
+          item.parent = self
+          self.save!
+          item.save!
+          self
+        end
+
+        alias :<< :add_item
+
+        # Dummy method. It is a placeholder for DB backed implementations. Wherever appropriate WorkItem#save will be
+        # called to save the current item's state. If state needs to persisted, you should override this method or make
+        # sure your persistence layer implements it in your class.
+        def save
+        end
+
+        # Dummy method. It is a placeholder for DB backed implementations. Wherever appropriate WorkItem#save will be
+        # called to save the current item's state. If state needs to persisted, you should override this method or make
+        # sure your persistence layer implements it in your class.
+        def save!
+        end
+
+        # Add a structured message to the log history. The message text can be submitted as an integer or text. If an
+        # integer is submitted, it will be used to look up the text in the MessageRegistry. The message text will be
+        # passed to the % operator with the args parameter. If that failes (e.g. because the format string is not correct)
+        # the args value is appended to the message.
+        #
+        # @param [Symbol] severity
+        # @param [Hash] msg should contain message text as :id or :text and the hierarchical name of the task as :task
+        # @param [Array] args string format values
+        def log_message(severity, msg, *args)
+          # Prepare info from msg struct for use with string substitution
+          message_id, message_text = if msg[:id]
+                                       [msg[:id], MessageRegistry.instance.get_message(msg[:id])]
+                                     elsif msg[:text]
+                                       [0, msg[:text]]
+                                     else
+                                       [0, '']
+                                     end
+          task = msg[:task] || '*UNKNOWN*'
+          message_text = (message_text % args rescue "#{message_text} - #{args}")
+
+          self.add_log severity: severity, id: message_id.to_i, text: message_text, task: task
+          name = ''
+          begin
+            name = self.to_s
+            name = self.name
+            name = self.namepath
+          rescue
+            # do nothing
+          end
+          Config.logger.add(severity, message_text, ('%s - %s ' % [task, name]))
+        end
+
+        protected
+
+        SEV_LABEL = %w(DEBUG INFO WARN ERROR FATAL ANY) unless const_defined? :SEV_LABEL
+
+        # go up the hierarchy and return the topmost work item
+        #
+        # @return [WorkItem] the root work item
+        def root
+          root = self
+          root = root.parent while root.parent and root.parent.is_a? WorkItem
+          root
+        end
+
+        # create and return a proper message structure
+        # @param [Hash] opts
+        def message_struct(opts = {})
+          opts.reverse_merge!(severity: ::Logger::INFO, code: nil, text: '')
+          {
+              severity: SEV_LABEL[opts[:severity]],
+              task: opts[:task],
+              code: opts[:code],
+              message: opts[:text]
+          }.cleanup
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/workflow/base/workflow.rb

@@ -5,12 +5,70 @@ require 'libis/tools/parameter'
 module Libis
   module Workflow
     module Base
+
+      # This is the base module for Workflows.
+      #
+      # This module lacks the implementation for the data attributes. It functions as an interface that describes the
+      # common functionality regardless of the storage implementation. These attributes require some implementation:
+      #
+      # - name: [String] the name of the Workflow. The name will be used to identify the workflow. Each time a workflow
+      #     is executed, a Run will be created. The Run will get a name that starts with the workflow name and ends with
+      #     the date and time the Run was started. As such this name attribute serves as an identifier and should be
+      #     treated as such. If possible is should be unique.
+      # - description: [String] more information about the workflow.
+      # - config: [Hash] detailed configuration for the workflow. The application assumes it behaves as a Hash and will
+      #     access it with [], merge! and delete methods. If your implementation decides to implement it with another
+      #     object, it should implement above methods. The config Hash requires the following keys:
+      #   - run_object: [String] the full class name of the Run implementation object that should be created when the
+      #       Workflow is executed.
+      #   - input: [Hash] all input parameter definitions where the key is the parameter name and the value is another
+      #       Hash with arguments for the parameter definition. It typically contains the following arguments:
+      #       - default: default value if no value specified when the workflow is executed
+      #       - propagate_to: the task name (or path) and parameter name that any set value for this parameter will be
+      #           propagated to. The syntax is <task name|task path>[#<parameter name>]. It the #<parameter name> part
+      #           is not present, the same name as the input parameter is used. If you want to push the value to
+      #           multiple task parameters, you can either supply an array of propagate paths or put them in a string
+      #           separated by a ','.
+      #   - tasks: [Array] task definitions that define the order in which the tasks should be executed for the workflow.
+      #       A task definition is a Hash with the following values:
+      #       - class: [String] the class name of the task including the module names
+      #       - name: [String] optional if class is present. A friendly name for the task that will be used in the logs.
+      #       - subitems: [Boolean] execute the task on the items in the current level or on the
+      #           child items of the current level. This parameter can be used in combination with the subtasks to
+      #           control what objects in the hierarchy the tasks are executed against.
+      #       - recursive: [Boolean] execute the task for the current level items only or automatically recurse through
+      #           the item's hierarchy and execute on all items below.
+      #       - tasks: [Array] a list of subtask defintions for this task.
+      #       Additionally the task definition Hash may specify values for any other parameter that the task knows of.
+      #       All tasks have parameters 'quiet', 'always_run', 'abort_on_error'. For more information about these see
+      #       the documentation of the task class.
+      #       A task definition does not require to have a 'class' entry. If not present the default
+      #       ::Libis::Workflow::Task class will be instatiated. It will do nothing itself, but will execute the
+      #       subtasks on the item(s). In such case a 'name' is mandatory.
+      #
+      # A minimal in-memory implementation could be:
+      #
+      # class Workflow
+      #   include ::Libis::Workflow::Base::Workflow
+      #
+      #   attr_accessor :name, :description, :config
+      #
+      #   def initialize
+      #     @name = ''
+      #     @descripition = ''
+      #     @config = Hash.new
+      #   end
+      #
+      # end
+      #
       module Workflow
 
         module ClassMethods
           def require_all
             Config.require_all(File.join(File.dirname(__FILE__), '..', 'tasks'))
+            # noinspection RubyResolve
             Config.require_all(Config.taskdir)
+            # noinspection RubyResolve
             Config.require_all(Config.itemdir)
           end
         end
@@ -19,20 +77,11 @@ module Libis
           base.extend ClassMethods
         end
 
-        def name; raise RuntimeError.new "Method not implemented: #{caller[0]}"; end
-        def name=(_) ; raise RuntimeError.new "Method not implemented: #{caller[0]}"; end
-
-        def description; raise RuntimeError.new "Method not implemented: #{caller[0]}"; end
-        def description=(_); raise RuntimeError.new "Method not implemented: #{caller[0]}"; end
-
-        def config; raise RuntimeError.new "Method not implemented: #{caller[0]}"; end
-        def config=(_); raise RuntimeError.new "Method not implemented: #{caller[0]}"; end
-
         def configure(cfg)
+          self.name = cfg.delete(:name) || self.class.name
+          self.description = cfg.delete(:description) || ''
           self.config.merge! input: {}, tasks: []
           self.config.merge! cfg
-          self.name = self.config.delete(:name) || self.class.name
-          self.description = self.config.delete(:description) || ''
 
           self.class.require_all
 
@@ -45,11 +94,15 @@ module Libis
 
         def input
           self.config[:input].inject({}) do |hash, input_def|
-            parameter = ::Libis::Tools::Parameter.new input_def.first.to_sym
+            name = input_def.first.to_sym
+            default = input_def.last[:default] || ''
+            parameter = ::Libis::Tools::Parameter.new name, default
             input_def.last.each { |k, v| parameter[k.to_sym] = v}
             hash[input_def.first.to_sym] = parameter
             hash
           end
+        rescue
+          {}
         end
 
         def run_name(timestamp = Time.now)
@@ -89,7 +142,7 @@ module Libis
             options[key] = parameter.parse(options[key])
             propagate_to = []
             propagate_to = parameter[:propagate_to] if parameter[:propagate_to].is_a? Array
-            propagate_to = [parameter[:propagate_to]] if parameter[:propagate_to].is_a? String
+            propagate_to = parameter[:propagate_to].split(/\s*,\s*/) if parameter[:propagate_to].is_a? String
             propagate_to.each do |target|
               task_name, param_name = target.split('#')
               param_name ||= key

data/lib/libis/workflow/base.rb

@@ -0,0 +1,6 @@
+require_relative 'base/logger'
+require_relative 'base/work_item'
+require_relative 'base/file_item'
+require_relative 'base/dir_item'
+require_relative 'base/run'
+require_relative 'base/workflow'

data/lib/libis/workflow/dir_item.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+require 'libis/workflow/file_item'
+
+module Libis
+  module Workflow
+
+    class DirItem < ::Libis::Workflow::FileItem
+    end
+
+  end
+end

data/lib/libis/workflow/file_item.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+
+require 'digest'
+
+require 'libis/workflow/base/file_item'
+require 'libis/workflow/work_item'
+
+module Libis
+  module Workflow
+
+    # noinspection RubyResolve
+    class FileItem < ::Libis::Workflow::WorkItem
+      include ::Libis::Workflow::Base::FileItem
+
+    end
+  end
+end

data/lib/libis/workflow/run.rb

@@ -4,20 +4,19 @@ require 'libis/workflow/config'
 require 'libis/workflow/workflow'
 
 require 'libis/workflow/base/run'
+require 'libis/workflow/work_item'
 
 module Libis
   module Workflow
 
-    class Run
+    class Run < ::Libis::Workflow::WorkItem
       include ::Libis::Workflow::Base::Run
 
-      attr_accessor :start_date, :tasks, :workflow
+      attr_accessor :start_date, :workflow
 
       def initialize
         @start_date = Time.now
-        @tasks = nil
         @workflow = nil
-        # noinspection RubySuperCallWithoutSuperclassInspection
         super
       end
 

data/lib/libis/workflow/task.rb

@@ -3,9 +3,9 @@ require 'backports/rails/hash'
 require 'backports/rails/string'
 
 require 'libis/tools/parameter'
+require 'libis/tools/extend/hash'
 
 require 'libis/workflow'
-require 'libis/workflow/base/logger'
 
 module Libis
   module Workflow
@@ -39,7 +39,7 @@ module Libis
 
       def run(item)
 
-        check_item_type WorkItem, item
+        check_item_type ::Libis::Workflow::Base::WorkItem, item
 
         return if item.failed? unless parameter(:always_run)
 
@@ -64,8 +64,9 @@ module Libis
           log_started item
 
           pre_process item
-          process_item item
-          post_process workitem
+          i = process_item item
+          item = i if i.is_a? Libis::Workflow::Base::WorkItem
+          post_process item
 
         rescue WorkflowError => e
           error e.message
@@ -92,14 +93,20 @@ module Libis
       def namepath; self.names.join('/'); end
 
       def apply_options(opts)
-        o = opts[self.name] || opts[self.names.join('/')]
-
-
-        default_values.each do |name,_|
-          next unless o.key?(name)
-          parameter = get_parameter_definition name
-          self.parameter(name, parameter.parse(o[name]))
-        end if o and o.is_a? Hash
+        o = {}
+        o.merge!(opts[self.class.to_s] || {})
+        o.merge!(opts[self.name] || opts[self.names.join('/')] || {})
+        o.key_strings_to_symbols! recursive: true
+
+        if o and o.is_a? Hash
+          default_values.each do |name, _|
+            next unless o.key?(name)
+            next unless o[name]
+            parameter = get_parameter_definition name
+            next unless (value = parameter.parse(o[name]))
+            self.parameter(name, value)
+          end
+        end
 
         self.tasks.each do |task|
           task.apply_options opts
@@ -110,16 +117,14 @@ module Libis
 
       def log_started(item)
         item.status = to_status :started
-        debug 'Started', item
       end
 
       def log_failed(item, message = nil)
-        warn (message || 'Failed'), item
+        warn (message), item if message
         item.status = to_status :failed
       end
 
       def log_done(item)
-        debug 'Completed', item
         item.status = to_status :done
       end
 
@@ -210,7 +215,7 @@ module Libis
             cfg[:options] || {}
         ).merge(
             cfg.reject { |k, _| [:options].include? k.to_sym }
-        ).symbolize_keys!.each { |k,v| parameter(k, v) }
+        ).symbolize_keys!.each { |k,v| self[k] = v }
       end
 
       def to_status(text)
@@ -248,7 +253,7 @@ module Libis
       end
 
       def self.default_values
-        parameters.inject({}) do |hash,parameter|
+        parameter_defs.inject({}) do |hash,parameter|
           hash[parameter.first] = parameter.last[:default]
           hash
         end

data/lib/libis/workflow/tasks/analyzer.rb

@@ -8,8 +8,11 @@ module Libis
 
       class Analyzer < Task
 
-        parameter quiet: true
-        parameter always_run: true
+        parameter quiet: true, frozen: true
+        parameter abort_on_error: false, frozen: true
+        parameter always_run: true, frozen: true
+        parameter subitems: false, frozen: true
+        parameter recursive: false, frozen: true
 
         def run(item)
 
@@ -30,7 +33,7 @@ module Libis
             end
           end
 
-        rescue Exception => ex
+        rescue RuntimeError => ex
 
           puts 'Failed to analyze item: %s - %s' % [item.class, item.name]
           puts 'Exception: %s' % ex.message

data/lib/libis/workflow/version.rb

@@ -2,6 +2,6 @@
 
 module Libis
   module Workflow
-    VERSION = '2.0.beta.9' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
+    VERSION = '2.0.beta.10' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
   end
 end

data/lib/libis/workflow/work_item.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require 'libis/tools/extend/hash'
+require 'libis/workflow/base/work_item'
+
+module Libis
+  module Workflow
+
+    # In-memory implementation of ::Libis::Workflow::Base::WorkItem
+    class WorkItem
+      include ::Libis::Workflow::Base::WorkItem
+
+      attr_accessor :parent
+      attr_accessor :items
+      attr_accessor :options, :properties
+      attr_accessor :log_history, :status_log
+      attr_accessor :summary
+
+      def initialize
+        self.parent = nil
+        self.items = []
+        self.options = {}
+        self.properties = {}
+        self.log_history = []
+        self.status_log = []
+        self.summary = {}
+      end
+
+      protected
+
+      def add_log_entry(msg)
+        # noinspection RubyResolve
+        self.log_history << msg.merge(c_at: ::Time.now)
+      end
+
+      def add_status_log(message, tasklist = nil)
+        # noinspection RubyResolve
+        self.status_log << {
+            timestamp: ::Time.now,
+            tasklist: tasklist,
+            text: message
+        }.cleanup
+      end
+
+      def status_label(status_entry)
+        "#{status_entry[:tasklist].last rescue nil}#{status_entry[:text] rescue nil}"
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow.rb

@@ -7,16 +7,23 @@ module Libis
     autoload :MessageRegistry, 'libis/workflow/message_registry'
     autoload :Config, 'libis/workflow/config'
 
-    autoload :WorkItem, 'libis/workflow/workitems/work_item'
-    autoload :FileItem, 'libis/workflow/workitems/file_item'
-    autoload :DirItem, 'libis/workflow/workitems/dir_item'
+    module Base
+      autoload :WorkItem, 'libis/workflow/base/work_item'
+      autoload :FileItem, 'libis/workflow/base/file_item'
+      autoload :DirItem, 'libis/workflow/base/dir_item'
+      autoload :Logger, 'libis/workflow/base/logger'
+      autoload :Run, 'libis/workflow/base/run'
+      autoload :Workflow, 'libis/workflow/base/workflow'
+    end
+
+    autoload :WorkItem, 'libis/workflow/work_item'
+    autoload :FileItem, 'libis/workflow/file_item'
+    autoload :DirItem, 'libis/workflow/dir_item'
 
     autoload :Workflow, 'libis/workflow/workflow'
     autoload :Run, 'libis/workflow/run'
     autoload :Task, 'libis/workflow/task'
 
-    autoload :Parameter, 'libis/workflow/parameter'
-
     autoload :Worker, 'libis/workflow/worker'
 
     def self.configure
@@ -25,4 +32,3 @@ module Libis
 
   end
 end
-

data/spec/items/test_dir_item.rb

@@ -1,8 +1,7 @@
 # encoding: utf-8
-require 'libis/workflow/workitems'
+require 'libis/workflow/dir_item'
 
-class TestDirItem
-  include ::Libis::Workflow::DirItem
+class TestDirItem < ::Libis::Workflow::DirItem
 
   def name=(dir)
     raise RuntimeError, "'#{dir}' is not a directory" unless File.directory? dir

data/spec/items/test_file_item.rb

@@ -1,10 +1,9 @@
 # encoding: utf-8
 require 'libis/tools/checksum'
 
-require 'libis/workflow/workitems'
+require 'libis/workflow/file_item'
 
-class TestFileItem
-  include ::Libis::Workflow::FileItem
+class TestFileItem < ::Libis::Workflow::FileItem
 
   def filename=(file)
     raise RuntimeError, "'#{file}' is not a file" unless File.file? file

data/spec/items/test_run.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require 'libis/workflow/workitems'
+require 'libis/workflow/run'
 
 require_relative 'test_dir_item'