libis-workflow 2.0.11 → 2.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db408853eaf1375de415370047728a3e4667d3f8
-  data.tar.gz: bc1996868fe1402e3a215d3b4ba3f55ff0e4886f
+  metadata.gz: f79b538ba0bf95e113d0f5fcf24abf11d924436a
+  data.tar.gz: 93e10a182e7788367a63f5e520b861027d6eea68
 SHA512:
-  metadata.gz: 1f19b3c2e59d474dc9d89698ea008128a1bc942c70686827f8c0631d37f2858d4d1d8cd2234bb25a725a6c5a7597f50fcfb7ab342eac42e279731c28e95b4aa0
-  data.tar.gz: 4673df7b1e87f8b4d79db363456f3bfb28e26142ecf7ab4a92cf7ff9b593346e47090eeee35a2860a1344ead49554bf18dee9b1d10763ecab9d19e5edbbb08e0
+  metadata.gz: d71d3a5142ecae99d83f35dab0efcc94ecc693ec2117f4970efa3a46d3c0f11ec423a77af8ad8d4efb1ace566e2ad7716538a6bfcf77f8483f59879f1dbe0f1d
+  data.tar.gz: 9baa84b4a68faedbff5723123ce1dc2c8863cce3558bfe19c7ec5b1f9a528cad0626890f4b38b6d4a2f1761c66c18d32000f995cfb42795ae2c38ea3e8ad80c5

data/README.md

@@ -50,10 +50,6 @@ name has to be provided to the job configuration so that the job can instantiate
 will be able to execute the tasks in proper order on all the WorkItems supplied/collected. Each task can be implemented 
 with code to run or simply contain a list of child tasks. 
 
-One task is predefined:
-::Libis::Workflow::Tasks::Analyzer - analyzes the workflow run and summarizes the results. It is always included as the
-last task by the workflow unless you supply a closing task called 'Analyzer' yourself.
-
 The whole ingester workflow is configured by a Singleton object ::Libis::Workflow::Config which contains settings for
 logging and paths where tasks and workitems can be found.
 
@@ -297,12 +293,6 @@ You have some options to control how the task will behave in special cases. Thes
 the task, which can be set (and fixed with the 'frozen' option) on the task, but can be configured at run-time with the 
 help of workflow input parameters and run options.
 
-#### Preventing any logging output from the task
-
-Logging output can be blocked on a task-by-task basis by setting the 'quiet' parameter to true. Probably not usefull 
-except for the Analyzer task where the parameter is fixed to true. Logging output would otherwise intervene with the 
-log summary processing performed by the task.
-
 #### Performing an action on the work item and all child items recursively
 
 With the 'recursive' parameter set to true, your task's process_item method will be called for the work item and then 

data/lib/libis/workflow.rb

@@ -29,6 +29,7 @@ module Libis
     autoload :Run, 'libis/workflow/run'
     autoload :Task, 'libis/workflow/task'
     autoload :TaskGroup, 'libis/workflow/task_group'
+    autoload :TaskRunner, 'libis/workflow/task_runner'
 
     autoload :Worker, 'libis/workflow/worker'
 

data/lib/libis/workflow/base/run.rb

@@ -1,6 +1,7 @@
 require 'fileutils'
 
 require 'libis/workflow/base/work_item'
+require 'libis/workflow/task_runner'
 
 module Libis
   module Workflow
@@ -72,10 +73,14 @@ module Libis
 
           self.save!
 
+          runner = Libis::Workflow::TaskRunner.new nil
+
           self.tasks.each do |task|
-            task.run self
+            runner << task
           end
 
+          runner.run self
+
         end
 
         protected

data/lib/libis/workflow/base/workflow.rb

@@ -84,12 +84,6 @@ module Libis
 
           self.class.require_all
 
-          unless !self.config['tasks'].empty? &&
-              self.config['tasks'].last['class'] &&
-              self.config['tasks'].last['class'].split('::').last == 'Analyzer'
-            self.config['tasks'] << {'class' => '::Libis::Workflow::Tasks::Analyzer'}
-          end
-
           self.config
         end
 

data/lib/libis/workflow/status.rb

@@ -11,6 +11,15 @@ module Libis
           FAILED: 5
       }
 
+      STATUS_TEXT = [
+          'not started',
+          'started',
+          'done',
+          'waiting for running async process',
+          'waiting for halted async process',
+          'failed'
+      ]
+
       # Changes the status of the object. The status changed is logged in the status_log with the current timestamp.
       #
       # @param [Array] x Array with status and task
@@ -29,6 +38,15 @@ module Libis
         status_symbol(entry[:status]) rescue :NOT_STARTED
       end
 
+      # Get last known status text for a given task
+      #
+      # @param [String] task task name to check item status for
+      # @return [Symbol] the status code
+      def status_text(task = nil)
+        entry = status_entry(task)
+        status_string(entry[:status]) rescue STATUS_TEXT.first
+      end
+
       # Gets the last known status label of the object.
       #
       # @param [String] task name of task to get the status for
@@ -78,6 +96,13 @@ module Libis
         STATUS.has_key?(x) ? x : nil
       end
 
+      def status_string(x)
+        return STATUS_TEXT[x] if x.is_a?(Integer)
+        return STATUS_TEXT[STATUS[x]] if STATUS.has_key?(x)
+        x = x.to_s.upcase.to_sym
+        STATUS.has_key?(x) ? STATUS_TEXT[STATUS[x]] : 'unknown status'
+      end
+
     end
   end
 end

data/lib/libis/workflow/task_group.rb

@@ -18,6 +18,7 @@ module Libis
 
       def <<(task)
         self.tasks << task
+        task.parent = self
       end
 
       def apply_options(opts)
@@ -45,6 +46,8 @@ module Libis
         end
 
         substatus_check(status, item, 'task')
+
+        info item.status_text(self.namepath).capitalize, item
       end
 
       def stop_processing_subtasks

data/lib/libis/workflow/task_runner.rb

@@ -0,0 +1,70 @@
+require_relative 'task_group'
+
+module Libis
+  module Workflow
+
+    # noinspection RubyTooManyMethodsInspection
+    class TaskRunner < Libis::Workflow::TaskGroup
+
+      parameter abort_on_failure: true, frozen: true
+      parameter recursive: false, frozen: true
+
+      def names
+        Array.new
+      end
+
+      def namepath
+        'Run'
+      end
+
+      def run(item)
+
+        check_item_type ::Libis::Workflow::Base::WorkItem, item
+        self.workitem = item
+
+        info 'Ingest run started.', item
+
+        super
+
+        recursive_run(item)
+        self.workitem = item
+
+      end
+
+      private
+
+      def recursive_run(item)
+
+        item.properties['ingest_failed'] = item.check_status(:FAILED)
+
+        item.summary = {}
+        item.log_history.each do |log|
+          level = log[:severity]
+          item.summary[level.to_s] ||= 0
+          item.summary[level.to_s] += 1
+        end
+
+        item.each do |i|
+          recursive_run i
+          i.summary.each do |level, count|
+            item.summary[level] ||= 0
+            item.summary[level] += (count || 0)
+          end
+        end
+
+      rescue RuntimeError => ex
+
+        self.workitem = item
+        error 'Failed to analyze item: %s - %s', item, item.class, item.name
+        error 'Exception: %s', item, ex.message
+
+      ensure
+
+        item.save!
+
+      end
+
+    end
+
+  end
+end

data/lib/libis/workflow/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module Workflow
-    VERSION = '2.0.11' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
+    VERSION = '2.0.12' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
   end
 end

data/spec/workflow_spec.rb

@@ -57,9 +57,9 @@ describe 'TestWorkflow' do
   }
 
   it 'should contain three tasks' do
-    expect(workflow.config['tasks'].size).to eq 3
+    expect(workflow.config['tasks'].size).to eq 2
     expect(workflow.config['tasks'].first['class']).to eq 'CollectFiles'
-    expect(workflow.config['tasks'].last['class']).to eq '::Libis::Workflow::Tasks::Analyzer'
+    expect(workflow.config['tasks'].last['name']).to eq 'ProcessFiles'
   end
 
   # noinspection RubyResolve
@@ -83,12 +83,15 @@ describe 'TestWorkflow' do
   it 'should return expected debug output' do
 
     sample_out = <<STR
+ INFO -- Run - TestRun : Ingest run started.
+ INFO -- Run - TestRun : Running subtask (1/2): CollectFiles
 DEBUG -- CollectFiles - TestRun : Processing subitem (1/1): items
 DEBUG -- CollectFiles - items : Processing subitem (1/3): test_dir_item.rb
 DEBUG -- CollectFiles - items : Processing subitem (2/3): test_file_item.rb
 DEBUG -- CollectFiles - items : Processing subitem (3/3): test_run.rb
 DEBUG -- CollectFiles - items : 3 of 3 subitems passed
 DEBUG -- CollectFiles - TestRun : 1 of 1 subitems passed
+ INFO -- Run - TestRun : Running subtask (2/2): ProcessFiles
  INFO -- ProcessFiles - TestRun : Running subtask (1/2): ChecksumTester
 DEBUG -- ProcessFiles/ChecksumTester - TestRun : Processing subitem (1/1): items
 DEBUG -- ProcessFiles/ChecksumTester - items : Processing subitem (1/3): test_dir_item.rb
@@ -103,7 +106,8 @@ DEBUG -- ProcessFiles/CamelizeName - Items : Processing subitem (2/3): test_file
 DEBUG -- ProcessFiles/CamelizeName - Items : Processing subitem (3/3): test_run.rb
 DEBUG -- ProcessFiles/CamelizeName - Items : 3 of 3 subitems passed
 DEBUG -- ProcessFiles/CamelizeName - TestRun : 1 of 1 subitems passed
- INFO -- Analyzer - TestRun : Ingest finished
+ INFO -- ProcessFiles - TestRun : Done
+ INFO -- Run - TestRun : Done
 STR
     sample_out = sample_out.lines.to_a
 
@@ -118,12 +122,13 @@ STR
     end
 
     expect(run.summary['DEBUG']).to eq 18
-    expect(run.log_history.size).to eq 9
-    expect(run.status_log.size).to eq 8
+    expect(run.log_history.size).to eq 13
+    expect(run.status_log.size).to eq 10
     expect(run.items.first.log_history.size).to eq 12
     expect(run.items.first.status_log.size).to eq 6
 
     [
+        {task: 'Run', status: :STARTED},
         {task: 'CollectFiles', status: :STARTED},
         {task: 'CollectFiles', status: :DONE},
         {task: 'ProcessFiles', status: :STARTED},
@@ -132,6 +137,7 @@ STR
         {task: 'ProcessFiles/CamelizeName', status: :STARTED},
         {task: 'ProcessFiles/CamelizeName', status: :DONE},
         {task: 'ProcessFiles', status: :DONE},
+        {task: 'Run', status: :DONE},
     ].each_with_index do |h, i|
       h.keys.each { |key| expect(run.status_log[i][key]).to eq h[key] }
     end
@@ -159,14 +165,19 @@ STR
     end
 
     [
+        {severity: 'INFO', task: 'Run', message: 'Ingest run started.'},
+        {severity: 'INFO', task: 'Run', message: 'Running subtask (1/2): CollectFiles'},
         {severity: 'DEBUG', task: 'CollectFiles', message: 'Processing subitem (1/1): items'},
         {severity: 'DEBUG', task: 'CollectFiles', message: '1 of 1 subitems passed'},
+        {severity: 'INFO', task: 'Run', message: 'Running subtask (2/2): ProcessFiles'},
         {severity: 'INFO', task: 'ProcessFiles', message: 'Running subtask (1/2): ChecksumTester'},
         {severity: 'DEBUG', task: 'ProcessFiles/ChecksumTester', message: 'Processing subitem (1/1): items'},
         {severity: 'DEBUG', task: 'ProcessFiles/ChecksumTester', message: '1 of 1 subitems passed'},
         {severity: 'INFO', task: 'ProcessFiles', message: 'Running subtask (2/2): CamelizeName'},
         {severity: 'DEBUG', task: 'ProcessFiles/CamelizeName', message: 'Processing subitem (1/1): items'},
         {severity: 'DEBUG', task: 'ProcessFiles/CamelizeName', message: '1 of 1 subitems passed'},
+        {severity: 'INFO', task: 'ProcessFiles', message: 'Done'},
+        {severity: 'INFO', task: 'Run', message: 'Done'},
     ].each_with_index do |h, i|
       h.keys.each { |key| expect(run.log_history[i][key]).to eq h[key] }
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libis-workflow
 version: !ruby/object:Gem::Version
-  version: 2.0.11
+  version: 2.0.12
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-06 00:00:00.000000000 Z
+date: 2016-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -155,7 +155,7 @@ files:
 - lib/libis/workflow/status.rb
 - lib/libis/workflow/task.rb
 - lib/libis/workflow/task_group.rb
-- lib/libis/workflow/tasks/analyzer.rb
+- lib/libis/workflow/task_runner.rb
 - lib/libis/workflow/version.rb
 - lib/libis/workflow/work_item.rb
 - lib/libis/workflow/worker.rb

data/lib/libis/workflow/tasks/analyzer.rb

@@ -1,58 +0,0 @@
-require 'libis/workflow/task'
-
-module Libis
-  module Workflow
-    module Tasks
-
-      class Analyzer < Task
-
-        parameter quiet: true, frozen: true
-        parameter recursive: false, frozen: true
-
-        # @param [Libis::Workflow::Base::WorkItem] item
-        def run(item)
-
-          recursive_run(item)
-          self.workitem = item
-          info 'Ingest finished', item
-
-        end
-
-        private
-
-        def recursive_run(item)
-
-          item.properties['ingest_failed'] = item.check_status(:FAILED)
-
-          item.summary = {}
-          item.log_history.each do |log|
-            level = log[:severity]
-            item.summary[level.to_s] ||= 0
-            item.summary[level.to_s] += 1
-          end
-
-          item.each do |i|
-            recursive_run i
-            i.summary.each do |level, count|
-              item.summary[level] ||= 0
-              item.summary[level] += (count || 0)
-            end
-          end
-
-        rescue RuntimeError => ex
-
-          self.workitem = item
-          error 'Failed to analyze item: %s - %s', item, item.class, item.name
-          error 'Exception: %s', item, ex.message
-
-        ensure
-
-          item.save!
-
-        end
-
-      end
-
-    end
-  end
-end