libis-workflow 2.0.12 → 2.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f79b538ba0bf95e113d0f5fcf24abf11d924436a
-  data.tar.gz: 93e10a182e7788367a63f5e520b861027d6eea68
+  metadata.gz: 391fa63e8c66d517802a33270a23090e71861051
+  data.tar.gz: d78da40fa5ebdca1174805ec76aa3c5b4e62ddcf
 SHA512:
-  metadata.gz: d71d3a5142ecae99d83f35dab0efcc94ecc693ec2117f4970efa3a46d3c0f11ec423a77af8ad8d4efb1ace566e2ad7716538a6bfcf77f8483f59879f1dbe0f1d
-  data.tar.gz: 9baa84b4a68faedbff5723123ce1dc2c8863cce3558bfe19c7ec5b1f9a528cad0626890f4b38b6d4a2f1761c66c18d32000f995cfb42795ae2c38ea3e8ad80c5
+  metadata.gz: 355050d00564e301721e794c4ef4effdd3c8626ed1e98d7c572942dca121769c4baf4468513391b92d4e917f0d63d84e10294d4d7ec2f50d4ec3eb2155bc4eb5
+  data.tar.gz: 81a05c1532b13641e3d48f0a4f37bdedb0e0ec52fc4cfc5641aef51814792546dbd3baf24a6bee1b3534688ddeb8f53311e9481beab3027b2afeba4f22fee61d

data/README.md

@@ -44,7 +44,7 @@ implementation is provided in the class ::Libis::Workflow::Workflow for your con
 your own from.
 
 The Job class is responsible for instantiating a run-time workflow execution object - a Run - that captures the 
-configuration, logs and workitems generated while executing the tasks. Essential logic is provided in the module 
+configuration and workitems generated while executing the tasks. Essential logic is provided in the module 
 ::Libis::Workflow::Base::Run with a simple in-memory implementation in ::Libis::Workflow::Run. The run object's class 
 name has to be provided to the job configuration so that the job can instantiate the correct object. The run object 
 will be able to execute the tasks in proper order on all the WorkItems supplied/collected. Each task can be implemented 
@@ -99,15 +99,17 @@ is a Hash with:
   ::Libis::Tools::Parameter class for this.
   
 The ::Libis::Workflow::Task base class allready defines the following parameters:
-* quiet: Prevent generating log output. Default: false
 * recursive: Run the task on all subitems recursively. Default: false
+* abort_recursion_on_failure: Stop processing items recursively if one item fails. Default: false
 * retry_count: Number of times to retry the task. Default: 0
 * retry_interval: Number of seconds to wait between retries. Default: 10
 
 If 'class' is not present, the default '::Libis::Workflow::TaskGroup' with the given name will be instantiated, which 
-performs each sub-task on the item. If the task is configured to be recursive, it will iterate over the child items and
-perform each sub-task on each of the child items. If a 'class' value is given, an instance of that class will be created 
-and the task will be handed the work item to process on. See the chapter on 'Tasks' below for more information on tasks.
+performs each sub-task on the item.
+
+If the task is configured to be recursive, it will iterate over the child items and perform each sub-task on each of 
+the child items. If a 'class' value is given, an instance of that class will be created and the task will be handed 
+the work item to process on. See the chapter on 'Tasks' below for more information on tasks.
 
 Note that a task with custom processing will not execute sub-tasks. If you configured a processing task with subtasks
 an exception will be thrown when trying to execute the job.
@@ -165,7 +167,6 @@ or if a custom storage implementation is desired, a number of data items and met
       stored_attribute :items
       stored_attribute :options
       stored_attribute :properties
-      stored_attribute :log_history
       stored_attribute :status_log
       stored_attribute :summary
 
@@ -174,23 +175,14 @@ or if a custom storage implementation is desired, a number of data items and met
         self.items = []
         self.options = {}
         self.properties = {}
-        self.log_history = []
         self.status_log = []
         self.summary = {}
       end
 
       protected
 
-      def add_log_entry(msg)
-        self.log_history << msg.merge(c_at: ::Time.now)
-      end
-
-      def add_status_log(message, tasklist = nil)
-        self.status_log << { timestamp: ::Time.now, tasklist: tasklist, text: message }.cleanup
-      end
-
-      def status_label(status_entry)
-        "#{status_entry[:tasklist].last rescue nil}#{status_entry[:text] rescue nil}"
+      def add_status_log(info)
+        self.status_log << info
       end
 
     end

data/lib/libis/workflow/base/work_item.rb

@@ -14,11 +14,6 @@ module Libis
       # This module lacks the implementation for the data attributes. It functions as an interface that describes the
       # common functionality regardless of the storage implementation. These attributes require some implementation:
       #
-      # - status: [Symbol] the status field. Each task sets the status of the items it works on. Before starting processing
-      #     the status is set to "#{task_name}Started". After successfull processing it is set to "#{task_name}Done" and if
-      #     the task failed, it is set to "#{task_name}Failed". The status field can be used to perform real-time
-      #     monitoring, reporting and error-recovery or restart of the ingest.
-      #     The initial value for this attribute is :START.
       # - parent: [Object|nil] a link to a parent work item. Work items can be organized in any hierarchy you think is
       #     relevant for your workflow (e.g. directory[/directory...]/file/line or library/section/book/page). Of course
       #     hierarchies are not mandatory.
@@ -28,9 +23,6 @@ module Libis
       # - properties: [Hash] a set of properties, typically collected during the workflow processing and used to store
       #     final or intermediate resulst of tasks. The ::Lias::Ingester::FileItem module uses this attribute to store the
       #     properties (e.g. size, checksum, ...) of the file it represents.
-      # - log_history: [Array] a list of all logging messages collected for this work item. Whenever a task logs a message
-      #     it will automatically be registered for the work item that it is processing or for the work item that was
-      #     supplied as the first argument.
       # - status_log: [Array] a list of all status changes the work item went through.
       # - summary: [Hash] collected statistics about the ingest for the work item and its children. This structure will
       #     be filled in by the included task ::Lias::Ingester::Tasks::Analyzer wich is appended to the workflow by default.
@@ -41,7 +33,7 @@ module Libis
       # attr_accessor :parent
       # attr_accessor :items
       # attr_accessor :options, :properties
-      # attr_accessor :log_history, :status_log
+      # attr_accessor :status_log
       # attr_accessor :summary
       #
       # def initialize
@@ -49,21 +41,16 @@ module Libis
       #   self.items = []
       #   self.options = {}
       #   self.properties = {}
-      #   self.log_history = []
       #   self.status_log = []
       #   self.summary = {}
       # end
       #
       # protected
       #
-      # ## Methods below should be adapted to match the implementation of the log arrays
+      # ## Method below should be adapted to match the implementation of the status array
       #
-      # def add_log_entry(msg)
-      #   self.log_history << msg.merge(c_at: ::Time.now)
-      # end
-      #
-      # def add_status_log(message, tasklist = nil)
-      #   self.status_log << { c_at: ::Time.now, tasklist: tasklist, text: message }.cleanup
+      # def add_status_log(info)
+      #   self.status_log << info
       # end
       #
       #

data/lib/libis/workflow/base/workflow.rb

@@ -34,8 +34,8 @@ module Libis
       #       - tasks: [Array] a list of subtask defintions for this task.
       #
       #       Additionally the task definition Hash may specify values for any other parameter that the task knows of.
-      #       All tasks have parameters 'quiet', 'recursive', 'retry_count' and 'retry_interval'. For more
-      #       information about these see the documentation of the task class.
+      #       All tasks have some fixed parameters. For more information about these see the documentation of
+      #       the task class.
       #
       #       A task definition does not require to have a 'class' entry. If not present the default
       #       ::Libis::Workflow::TaskGroup class will be instatiated. It will do nothing itself, but will execute the

data/lib/libis/workflow/status.rb

@@ -22,10 +22,16 @@ module Libis
 
       # Changes the status of the object. The status changed is logged in the status_log with the current timestamp.
       #
-      # @param [Array] x Array with status and task
-      def status=(x)
-        s, task = x
-        self.add_status_log(task: task, status: s)
+      # @param [String] task namepath of the task
+      # @param [Symbol] status status to set
+      def set_status(task, status)
+        case status
+          when :STARTED
+            self.add_status_log(task: task, status: status)
+          else
+            log_entry = status_entry(task) || self.add_status_log(task: task, status: status)
+            log_entry[:status] = status
+        end
         self.save!
       end
 
@@ -73,6 +79,18 @@ module Libis
         STATUS[self.status(task)] <=> STATUS[state]
       end
 
+      # Update the progress of the working task
+      # @param [String] task namepath of the task
+      # @param [Integer] progress progress indicator (as <progress> of <max> or as % if <max> not set). Default: 0
+      # @param [Integer] max max count.
+      def status_progress(task, progress = 0, max = nil)
+        log_entry = self.status_entry(task)
+        log_entry ||= self.status_log.build(task: task)
+        log_entry[:progress] = progress
+        log_entry[:max] = max if max
+        self.save!
+      end
+
       protected
 
       # Get last known status entry for a given task

data/lib/libis/workflow/task.rb

@@ -17,8 +17,8 @@ module Libis
 
       attr_accessor :parent, :name, :workitem
 
-      parameter quiet: false, description: 'Prevent generating log output.'
       parameter recursive: false, description: 'Run the task on all subitems recursively.'
+      parameter abort_recursion_on_failure: false, description: 'Stop processing items recursively if one item fails.'
       parameter retry_count: 0, description: 'Number of times to retry the task if waiting for another process.'
       parameter retry_interval: 10, description: 'Number of seconds to wait between retries.'
 
@@ -124,23 +124,13 @@ module Libis
       def message(severity, msg, *args)
         taskname = self.namepath rescue nil
         self.set_application(taskname)
-        item = self.workitem
+        item = self.workitem rescue nil
         item = args.shift if args.size > 0 and args[0].is_a?(::Libis::Workflow::Base::WorkItem)
-        if item
-          subject = nil
-          begin
-            subject = item.to_s
-            subject = item.name
-            subject = item.namepath
-          rescue
-            # do nothing
-          end
-          self.set_subject(subject)
-          return unless super(severity, msg, *args)
-          item.log_message(
-              severity, msg.is_a?(Integer) ? {id: msg} : {text: (msg.to_s rescue '')}.merge(task: taskname), *args
-          )
-        end
+        subject = item.namepath rescue nil
+        subject ||= item.name rescue nil
+        subject ||= item.to_s rescue nil
+        self.set_subject(subject)
+        super(severity, msg, *args)
       end
 
       def logger
@@ -192,10 +182,14 @@ module Libis
         return unless items.size > 0
 
         status = Hash.new(0)
+        parent_item.status_progress(self.namepath, 0, items.count)
         items.each_with_index do |item, i|
           debug 'Processing subitem (%d/%d): %s', parent_item, i+1, items.size, item.to_s
           run_item item
-          status[item.status(self.namepath)] += 1
+          parent_item.status_progress(self.namepath, i+1)
+          item_status = item.status(self.namepath)
+          status[item_status] += 1
+          break if parameter(:abort_recursion_on_failure) && item_status != :DONE
         end
 
         debug '%d of %d subitems passed', parent_item, status[:DONE], items.size
@@ -275,14 +269,10 @@ module Libis
       end
 
       def set_status(item, state)
-        item.status = to_status(state)
+        item.set_status self.namepath, state
         state
       end
 
-      def to_status(state)
-        [state, self.namepath]
-      end
-
       def check_item_type(klass, item = nil)
         item ||= self.workitem
         unless item.is_a? klass.to_s.constantize

data/lib/libis/workflow/task_group.rb

@@ -38,11 +38,14 @@ module Libis
         return unless tasks.size > 0
 
         status = Hash.new(0)
+        item.status_progress(self.namepath, 0, tasks.count)
         tasks.each_with_index do |task, i|
           info 'Running subtask (%d/%d): %s', item, i+1, tasks.size, task.name
           task.run item
-          status[item.status(task.namepath)] += 1
-          break if parameter(:abort_on_failure) && item.status(task.namepath) != :DONE
+          item.status_progress(self.namepath, i+1)
+          item_status = item.status(task.namepath)
+          status[item_status] += 1
+          break if parameter(:abort_on_failure) && item_status != :DONE
         end
 
         substatus_check(status, item, 'task')

data/lib/libis/workflow/task_runner.rb

@@ -26,42 +26,6 @@ module Libis
 
         super
 
-        recursive_run(item)
-        self.workitem = item
-
-      end
-
-      private
-
-      def recursive_run(item)
-
-        item.properties['ingest_failed'] = item.check_status(:FAILED)
-
-        item.summary = {}
-        item.log_history.each do |log|
-          level = log[:severity]
-          item.summary[level.to_s] ||= 0
-          item.summary[level.to_s] += 1
-        end
-
-        item.each do |i|
-          recursive_run i
-          i.summary.each do |level, count|
-            item.summary[level] ||= 0
-            item.summary[level] += (count || 0)
-          end
-        end
-
-      rescue RuntimeError => ex
-
-        self.workitem = item
-        error 'Failed to analyze item: %s - %s', item, item.class, item.name
-        error 'Exception: %s', item, ex.message
-
-      ensure
-
-        item.save!
-
       end
 
     end

data/lib/libis/workflow/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module Workflow
-    VERSION = '2.0.12' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
+    VERSION = '2.0.13' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
   end
 end

data/lib/libis/workflow/work_item.rb

@@ -11,7 +11,7 @@ module Libis
       attr_accessor :parent
       attr_accessor :items
       attr_accessor :options, :properties
-      attr_accessor :log_history, :status_log
+      attr_accessor :status_log
       attr_accessor :summary
 
       def initialize
@@ -19,21 +19,15 @@ module Libis
         self.items = []
         self.options = {}
         self.properties = {}
-        self.log_history = []
         self.status_log = []
         self.summary = {}
       end
 
       protected
 
-      def add_log_entry(msg)
-        # noinspection RubyResolve
-        self.log_history << msg.merge(c_at: ::Time.now)
-      end
-
       def add_status_log(info)
         # noinspection RubyResolve
-        self.status_log << info.merge(timestamp: ::Time.now).cleanup
+        self.status_log << info
       end
 
     end

data/spec/workflow_spec.rb

@@ -121,87 +121,35 @@ STR
       expect(o[/(?<=\] ).*/].strip).to eq sample_out[i].strip
     end
 
-    expect(run.summary['DEBUG']).to eq 18
-    expect(run.log_history.size).to eq 13
-    expect(run.status_log.size).to eq 10
-    expect(run.items.first.log_history.size).to eq 12
-    expect(run.items.first.status_log.size).to eq 6
+    expect(run.status_log.size).to eq 5
+    expect(run.items.first.status_log.size).to eq 3
 
     [
-        {task: 'Run', status: :STARTED},
-        {task: 'CollectFiles', status: :STARTED},
-        {task: 'CollectFiles', status: :DONE},
-        {task: 'ProcessFiles', status: :STARTED},
-        {task: 'ProcessFiles/ChecksumTester', status: :STARTED},
-        {task: 'ProcessFiles/ChecksumTester', status: :DONE},
-        {task: 'ProcessFiles/CamelizeName', status: :STARTED},
-        {task: 'ProcessFiles/CamelizeName', status: :DONE},
-        {task: 'ProcessFiles', status: :DONE},
-        {task: 'Run', status: :DONE},
+        {task: 'Run', status: :DONE, progress: 2, max: 2},
+        {task: 'CollectFiles', status: :DONE, progress: 1, max: 1},
+        {task: 'ProcessFiles', status: :DONE, progress: 2, max: 2},
+        {task: 'ProcessFiles/ChecksumTester', status: :DONE, progress: 1, max: 1},
+        {task: 'ProcessFiles/CamelizeName', status: :DONE, progress: 1, max: 1},
     ].each_with_index do |h, i|
       h.keys.each { |key| expect(run.status_log[i][key]).to eq h[key] }
     end
 
     [
-        {task: 'CollectFiles', status: :STARTED},
-        {task: 'CollectFiles', status: :DONE},
-        {task: 'ProcessFiles/ChecksumTester', status: :STARTED},
-        {task: 'ProcessFiles/ChecksumTester', status: :DONE},
-        {task: 'ProcessFiles/CamelizeName', status: :STARTED},
-        {task: 'ProcessFiles/CamelizeName', status: :DONE},
+        {task: 'CollectFiles', status: :DONE, progress: 3, max: 3},
+        {task: 'ProcessFiles/ChecksumTester', status: :DONE, progress: 3, max: 3},
+        {task: 'ProcessFiles/CamelizeName', status: :DONE, progress: 3, max: 3},
     ].each_with_index do |h, i|
       h.keys.each { |key| expect(run.items.first.status_log[i][key]).to eq h[key] }
     end
 
     [
-        {task: 'CollectFiles', status: :STARTED},
-        {task: 'CollectFiles', status: :DONE},
-        {task: 'ProcessFiles/ChecksumTester', status: :STARTED},
-        {task: 'ProcessFiles/ChecksumTester', status: :DONE},
-        {task: 'ProcessFiles/CamelizeName', status: :STARTED},
-        {task: 'ProcessFiles/CamelizeName', status: :DONE},
+        {task: 'CollectFiles', status: :DONE, progress: nil, max: nil},
+        {task: 'ProcessFiles/ChecksumTester', status: :DONE, progress: nil, max: nil},
+        {task: 'ProcessFiles/CamelizeName', status: :DONE, progress: nil, max: nil},
     ].each_with_index do |h, i|
       h.keys.each { |key| expect(run.items.first.first.status_log[i][key]).to eq h[key] }
     end
 
-    [
-        {severity: 'INFO', task: 'Run', message: 'Ingest run started.'},
-        {severity: 'INFO', task: 'Run', message: 'Running subtask (1/2): CollectFiles'},
-        {severity: 'DEBUG', task: 'CollectFiles', message: 'Processing subitem (1/1): items'},
-        {severity: 'DEBUG', task: 'CollectFiles', message: '1 of 1 subitems passed'},
-        {severity: 'INFO', task: 'Run', message: 'Running subtask (2/2): ProcessFiles'},
-        {severity: 'INFO', task: 'ProcessFiles', message: 'Running subtask (1/2): ChecksumTester'},
-        {severity: 'DEBUG', task: 'ProcessFiles/ChecksumTester', message: 'Processing subitem (1/1): items'},
-        {severity: 'DEBUG', task: 'ProcessFiles/ChecksumTester', message: '1 of 1 subitems passed'},
-        {severity: 'INFO', task: 'ProcessFiles', message: 'Running subtask (2/2): CamelizeName'},
-        {severity: 'DEBUG', task: 'ProcessFiles/CamelizeName', message: 'Processing subitem (1/1): items'},
-        {severity: 'DEBUG', task: 'ProcessFiles/CamelizeName', message: '1 of 1 subitems passed'},
-        {severity: 'INFO', task: 'ProcessFiles', message: 'Done'},
-        {severity: 'INFO', task: 'Run', message: 'Done'},
-    ].each_with_index do |h, i|
-      h.keys.each { |key| expect(run.log_history[i][key]).to eq h[key] }
-    end
-
-    [
-        {severity: 'DEBUG', task: 'CollectFiles', message: 'Processing subitem (1/3): test_dir_item.rb'},
-        {severity: 'DEBUG', task: 'CollectFiles', message: 'Processing subitem (2/3): test_file_item.rb'},
-        {severity: 'DEBUG', task: 'CollectFiles', message: 'Processing subitem (3/3): test_run.rb'},
-        {severity: 'DEBUG', task: 'CollectFiles', message: '3 of 3 subitems passed'},
-        {severity: 'DEBUG', task: 'ProcessFiles/ChecksumTester', message: 'Processing subitem (1/3): test_dir_item.rb'},
-        {severity: 'DEBUG', task: 'ProcessFiles/ChecksumTester', message: 'Processing subitem (2/3): test_file_item.rb'},
-        {severity: 'DEBUG', task: 'ProcessFiles/ChecksumTester', message: 'Processing subitem (3/3): test_run.rb'},
-        {severity: 'DEBUG', task: 'ProcessFiles/ChecksumTester', message: '3 of 3 subitems passed'},
-        {severity: 'DEBUG', task: 'ProcessFiles/CamelizeName', message: 'Processing subitem (1/3): test_dir_item.rb'},
-        {severity: 'DEBUG', task: 'ProcessFiles/CamelizeName', message: 'Processing subitem (2/3): test_file_item.rb'},
-        {severity: 'DEBUG', task: 'ProcessFiles/CamelizeName', message: 'Processing subitem (3/3): test_run.rb'},
-        {severity: 'DEBUG', task: 'ProcessFiles/CamelizeName', message: '3 of 3 subitems passed'},
-    ].each_with_index do |h, i|
-      h.keys.each { |key| expect(run.items.first.log_history[i][key]).to eq h[key] }
-    end
-
-    # noinspection RubyResolve
-    expect(run.items.first.first.log_history).to be_empty
-
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libis-workflow
 version: !ruby/object:Gem::Version
-  version: 2.0.12
+  version: 2.0.13
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-13 00:00:00.000000000 Z
+date: 2016-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler