libis-workflow 2.0.beta.3

data/lib/libis/workflow/workitems/work_item.rb

@@ -0,0 +1,231 @@
+# encoding: utf-8
+
+require 'backports/rails/hash'
+
+require 'libis/workflow/config'
+
+module Libis
+  module Workflow
+
+    # Base module for all work items.
+    #
+    # This module contains some basic attributes required for making the workflow and tasks behave properly:
+    #
+    # - status: [Symbol] the status field. Each task sets the status of the items it works on. Before starting processing
+    #     the status is set to "#{task_name}Started". After successfull processing it is set to "#{task_name}Done" and if
+    #     the task failed, it is set to "#{task_name}Failed". The status field can be used to perform real-time
+    #     monitoring, reporting and error-recovery or restart of the ingest.
+    #     The initial value for this attribute is :START.
+    # - parent: [Object|nil] a link to a parent work item. Work items can be organized in any hierarchy you think is
+    #     relevant for your workflow (e.g. directory[/directory...]/file/line or library/section/book/page). Of course
+    #     hierarchies are not mandatory.
+    # - items: [Array] a list of child work items. see above.
+    # - options: [Hash] a set of options for the task chain on how to deal with this work item. This attribute can be
+    #     used to fine-tune the behaviour of tasks for a particular work item.
+    # - properties: [Hash] a set of properties, typically collected during the workflow processing and used to store
+    #     final or intermediate resulst of tasks. The ::Lias::Ingester::FileItem module uses this attribute to store the
+    #     properties (e.g. size, checksum, ...) of the file it represents.
+    # - log_history: [Array] a list of all logging messages collected for this work item. Whenever a task logs a message
+    #     it will automatically be registered for the work item that it is processing or for the work item that was
+    #     supplied as the first argument.
+    # - status_log: [Array] a list of all status changes the work item went through.
+    # - summary: [Hash] collected statistics about the ingest for the work item and its children. This structure will
+    #     be filled in by the included task ::Lias::Ingester::Tasks::Analyzer wich is appended to the workflow by default.
+    #
+    # The module is created so that it is possible to implement an ActiveRecord/Datamapper/... implementation easily.
+
+    module WorkItem
+      include Enumerable
+
+      attr_accessor :parent
+      attr_accessor :items
+      attr_accessor :options, :properties
+      attr_accessor :log_history, :status_log
+      attr_accessor :summary
+
+      # The initializer takes care of properly setting the correct default values for the attributes. A derived class
+      # that wishes to define it's own initializer should take care to call 'super' or make sure it overwrites the
+      # attribute definitions itself. (e.g. in a ActiveRecord implementation)
+      def initialize
+        self.parent = nil
+        self.items = []
+        self.options = {}
+        self.properties = {}
+        self.log_history = []
+        self.status_log = []
+        self.summary = {}
+      end
+
+      # String representation of the identity of the work item.
+      #
+      # You may want to overwrite this method as it tries the :name property or whatever #inspect returns if that
+      # failes. Typically this should return the key value, file name or id number. If that's what your :name property
+      # contains, you're fine.
+      #
+      # @return [String] string identification for this work item.
+      def name
+        self.properties[:name] || self.inspect
+      end
+
+      def to_s; self.name; end
+
+      def names
+        (self.parent.names rescue Array.new).push(name).compact
+      end
+
+      def namepath; self.names.join('/'); end
+
+      # File name save version of the to_s output. The output should be safe to use as a file name to store work item
+      # data. Typical use is when extra file items are created by a task and need to be stored on disk. The default
+      # implementation URL-encodes (%xx) all characters except alphanumeric, '.' and '-'.
+      #
+      # @return [String] file name
+      def to_filename
+        self.to_s.gsub(/[^\w.-]/) { |s| '%%%02x' % s.ord }
+      end
+
+      # Gets the current status of the object.
+      #
+      # @return [Symbol] status code
+      def status
+        s = self.status_log.last
+        status_label((s[:text] rescue nil), (s[:tasklist] rescue nil))
+      end
+
+      # Changes the status of the object. As a side effect the status is also logged in the status_log with the current
+      # timestamp.
+      #
+      # @param [Symbol] s
+      def status=(s, tasklist = nil)
+        s, tasklist = s if s.is_a? Array
+        s = s.to_sym
+        if status_label(s, tasklist) != self.status
+          self.status_log << {
+              timestamp: ::Time.now,
+              tasklist: tasklist,
+              text: s
+          }
+          self.save
+        end
+      end
+
+      # Check ingest status of the object. The status is checked to see if it ends in 'Failed'.
+      #
+      # @return [Boolean] true if the object failed, false otherwise
+      def failed?
+        self.status.to_s =~ /Failed$/ ? true : false
+      end
+
+      # Helper function for the Tasks to add a log entry to the log_history.
+      #
+      # The supplied message structure is expected to contain the following fields:
+      # - :severity : ::Logger::Severity value
+      # - :id : optional message id
+      # - :text : message text
+      # - :task : list of tasks names (task hierarchy) that submits the message
+      #
+      # @param [Hash] message
+      def add_log(message = {})
+        msg = message_struct(message)
+        self.log_history << msg
+        self.save
+      end
+
+      def <=(message = {}); self.add_log(message); end
+
+      # Iterates over the work item clients and invokes code on each of them.
+      def each
+        self.items.each { |item| yield item }
+      end
+
+      # Add a child work item
+      #
+      # @param [WorkItem] item to be added to the child list :items
+      def add_item(item)
+        return self unless item and item.is_a? WorkItem
+        self.items << item
+        item.parent = self
+        self.save!
+        item.save!
+        self
+      end
+
+      alias :<< :add_item
+
+      # Dummy method. It is a placeholder for DB backed implementations. Wherever appropriate WorkItem#save will be
+      # called to save the current item's state. If state needs to persisted, you should override this method or make
+      # sure your persistence layer implements it in your class.
+      def save
+      end
+
+      # Dummy method. It is a placeholder for DB backed implementations. Wherever appropriate WorkItem#save will be
+      # called to save the current item's state. If state needs to persisted, you should override this method or make
+      # sure your persistence layer implements it in your class.
+      def save!
+      end
+
+      # Add a structured message to the log history. The message text can be submitted as an integer or text. If an
+      # integer is submitted, it will be used to look up the text in the MessageRegistry. The message text will be
+      # passed to the % operator with the args parameter. If that failes (e.g. because the format string is not correct)
+      # the args value is appended to the message.
+      #
+      # @param [Symbol] severity
+      # @param [Hash] msg should contain message text as :id or :text and the hierarchical name of the task as :task
+      # @param [Array] args string format values
+      def log_message(severity, msg, *args)
+        # Prepare info from msg struct for use with string substitution
+        message_id, message_text = if msg[:id]
+                                     [msg[:id], MessageRegistry.instance.get_message(msg[:id])]
+                                   elsif msg[:text]
+                                     [0, msg[:text]]
+                                   else
+                                     [0, '']
+                                   end
+        task = msg[:task] || '*UNKNOWN*'
+        message_text = (message_text % args rescue "#{message_text} - #{args}")
+
+        self.add_log severity: severity, id: message_id.to_i, text: message_text, task: task
+        name = ''
+        begin
+          name = self.to_s
+          name = self.name
+          name = self.namepath
+        rescue
+          # do nothing
+        end
+        Config.logger.add(severity, message_text, ('%s - %s ' % [task, name]))
+      end
+
+      protected
+
+      SEV_LABEL = %w(DEBUG INFO WARN ERROR FATAL ANY) unless const_defined? :SEV_LABEL
+
+      # go up the hierarchy and return the topmost work item
+      #
+      # @return [WorkItem] the root work item
+      def root
+        root = self
+        root = root.parent while root.parent and root.parent.is_a? WorkItem
+        root
+      end
+
+      # create and return a proper message structure
+      # @param [Hash] opts
+      def message_struct(opts = {})
+        opts.reverse_merge!(severity: ::Logger::INFO, id: 0, text: '')
+        {
+            timestamp: ::Time.now,
+            severity: SEV_LABEL[opts[:severity]],
+            task: opts[:task],
+            id: opts[:id],
+            message: opts[:text]
+        }
+      end
+
+      def status_label(text, tasklist)
+        s = "#{tasklist.last rescue ''}#{text}" rescue :NOT_STARTED
+        s.empty? ? :NOT_STARTED : s.to_sym
+      end
+    end
+  end
+end

data/lib/libis/workflow/workitems.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+require_relative 'workitems/work_item'
+require_relative 'workitems/file_item'
+require_relative 'workitems/dir_item'

data/lib/libis/workflow.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+require 'libis/exceptions'
+
+module Libis
+  module Workflow
+
+    autoload :MessageRegistry, 'libis/workflow/message_registry'
+    autoload :Config, 'libis/workflow/config'
+
+    autoload :WorkItem, 'libis/workflow/workitems/work_item'
+    autoload :FileItem, 'libis/workflow/workitems/file_item'
+    autoload :DirItem, 'libis/workflow/workitems/dir_item'
+
+    autoload :Workflow, 'libis/workflow/workflow'
+    autoload :Run, 'libis/workflow/run'
+    autoload :Task, 'libis/workflow/task'
+
+    autoload :Parameter, 'libis/workflow/parameter'
+
+    autoload :Worker, 'libis/workflow/worker'
+
+    def self.configure
+      yield Config.instance
+    end
+
+  end
+end
+

data/lib/libis-workflow.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require 'libis/workflow'

data/libis-workflow.gemspec

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'libis/workflow/version'
+
+Gem::Specification.new do |gem|
+  gem.name = 'libis-workflow'
+  gem.version = ::Libis::Workflow::VERSION
+  gem.date = Date.today.to_s
+
+  gem.summary = %q{LIBIS Workflow framework.}
+  gem.description = %q{A simple framework to build custom task/workflow solutions.}
+
+  gem.author = 'Kris Dekeyser'
+  gem.email = 'kris.dekeyser@libis.be'
+  gem.homepage = 'https://github.com/Kris-LIBIS/workflow'
+  gem.license = 'MIT'
+
+  gem.files = `git ls-files -z`.split("\x0")
+  gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
+
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency 'bundler', '~> 1.6'
+  gem.add_development_dependency 'rake', '~> 10.3'
+  gem.add_development_dependency 'rspec', '~> 3.1'
+  gem.add_development_dependency 'simplecov', '~> 0.9'
+  gem.add_development_dependency 'coveralls', '~> 0.7'
+
+  gem.add_runtime_dependency 'libis-tools', '~> 0.9'
+  gem.add_runtime_dependency 'sidekiq', '~> 3.3'
+
+end

data/spec/items/test_dir_item.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require 'libis/workflow/workitems'
+
+class TestDirItem
+  include ::Libis::Workflow::DirItem
+
+  def name=(dir)
+    raise RuntimeError, "'#{dir}' is not a directory" unless File.directory? dir
+    super dir
+  end
+
+  def name
+    self.properties[:name] || super
+  end
+
+end

data/spec/items/test_file_item.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+require 'libis/tools/checksum'
+
+require 'libis/workflow/workitems'
+
+class TestFileItem
+  include ::Libis::Workflow::FileItem
+
+  def filename=(file)
+    raise RuntimeError, "'#{file}' is not a file" unless File.file? file
+    set_checksum :SHA256, ::Libis::Tools::Checksum.hexdigest(file, :SHA256)
+    super file
+  end
+
+  def name
+    self.properties[:name] || super
+  end
+
+end

data/spec/items/test_run.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require 'libis/workflow/workitems'
+
+require_relative 'test_dir_item'
+
+class TestRun < ::Libis::Workflow::Run
+
+  def name; 'TestRun'; end
+
+end

data/spec/items.rb

@@ -0,0 +1,3 @@
+require_relative 'items/test_dir_item'
+require_relative 'items/test_file_item'
+require_relative 'items/test_run'

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'coveralls'
+Coveralls.wear!
+
+require 'bundler/setup'
+Bundler.setup
+
+require 'rspec'
+require 'libis-workflow'

data/spec/task_spec.rb

@@ -0,0 +1,17 @@
+require_relative 'spec_helper'
+
+require 'libis/workflow/task'
+
+describe 'Task' do
+
+  it 'should create a default task' do
+
+    task = ::Libis::Workflow::Task.new nil
+
+    expect(task.parent).to eq nil
+    expect(task.name).to eq 'Task'
+    expect(task.options[:abort_on_error]).to eq false
+
+  end
+
+end

data/spec/tasks/camelize_name.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require 'backports/rails/string'
+
+require 'libis/workflow/workitems'
+
+class CamelizeName < ::Libis::Workflow::Task
+
+  def process(item)
+    return unless (item.is_a?(TestFileItem) || item.is_a?(TestDirItem))
+    item.properties[:name] = item.name.camelize
+  end
+
+end

data/spec/tasks/checksum_tester.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require 'libis/tools/checksum'
+
+require 'libis/exceptions'
+require 'libis/workflow/workitems'
+
+class ChecksumTester < ::Libis::Workflow::Task
+
+  parameter checksum_type: nil,
+            description: 'Checksum type to use.',
+            constraint: ::Libis::Tools::Checksum::CHECKSUM_TYPES.map {|x| x.to_s}
+
+  def process(item)
+    return unless item.is_a? TestFileItem
+
+    checksum_type = options[:checksum_type]
+
+    if checksum_type.nil?
+      ::Libis::Tools::Checksum::CHECKSUM_TYPES.each do |x|
+        test_checksum(item, x) if item.checksum(x)
+      end
+    else
+      test_checksum(item, checksum_type)
+    end
+  end
+
+  def test_checksum(item, checksum_type)
+    checksum = ::Libis::Tools::Checksum.hexdigest(item.fullpath, checksum_type.to_sym)
+    return if item.checksum(checksum_type) == checksum
+    raise ::Libis::WorkflowError, "Checksum test #{checksum_type} failed for #{item.filepath}"
+  end
+
+end

data/spec/tasks/collect_files.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require 'libis/exceptions'
+
+require_relative '../items'
+
+class CollectFiles < ::Libis::Workflow::Task
+
+  parameter location: '.',
+            description: 'Dir location to start scanning for files.'
+  parameter subdirs: false,
+            description: 'Look for files in subdirs too.'
+  parameter selection: nil,
+            description: 'Only select files that match the given regular expression. Ignored if empty.'
+
+  def process(item)
+    if item.is_a? TestRun
+      add_item(item, options[:location])
+    elsif item.is_a? TestDirItem
+      collect_files(item, item.fullpath)
+    end
+  end
+
+  def collect_files(item, dir)
+    glob_string = dir
+    glob_string = File.join(glob_string, '**') if options[:subdirs]
+    glob_string = File.join(glob_string, '*')
+
+    Dir.glob(glob_string).select do |x|
+      options[:selection] && !options[:selection].empty? ? x =~ Regexp.new(options[:selection]) : true
+    end.sort.each do |file|
+      next if %w'. ..'.include? file
+      add_item(item, file)
+    end
+  end
+
+  def add_item(item, file)
+    child = if File.file?(file)
+              TestFileItem.new
+            elsif File.directory?(file)
+              TestDirItem.new
+            else
+              Item.new
+            end
+    child.filename = file
+    item << child
+  end
+
+end