libis-workflow-activerecord 0.9.0

data/lib/libis/workflow/activerecord/worker.rb

@@ -0,0 +1,20 @@
+require 'libis/workflow/worker'
+require 'sidekiq'
+
+module Libis
+  module Workflow
+    module ActiveRecord
+
+      class Worker < Libis::Workflow::Worker
+
+        def get_job(job_config)
+          job_name = job_config.delete(:name)
+          job = ::Libis::Workflow::ActiveRecord::Job.find(name: job_name).first
+          raise RuntimeError.new "Workflow #{job_name} not found" unless job.is_a? ::Libis::Workflow::ActiveRecord::Job
+          job
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/workflow/activerecord/workflow.rb

@@ -0,0 +1,40 @@
+require 'libis/workflow/activerecord'
+require 'libis/workflow/base/workflow'
+require 'libis/tools/config_file'
+require 'libis/tools/extend/hash'
+
+module Libis
+  module Workflow
+    module ActiveRecord
+
+      class Workflow < ::ActiveRecord::Base
+        include ::Libis::Workflow::Base::Workflow
+        include ::Libis::Workflow::ActiveRecord::Base
+
+        # noinspection RailsParamDefResolve
+        has_many :jobs,
+                 -> {order('id')},
+                 class_name: Libis::Workflow::ActiveRecord::Job.to_s,
+                 foreign_key: :workflow_id,
+                 autosave: true
+
+        def self.from_hash(hash)
+          self.create_from_hash(hash, [:name]) do |item, cfg|
+            item.configure(cfg.merge('name' => item.name))
+            cfg.clear
+          end
+        end
+
+        def self.load(file_or_hash)
+          config = Libis::Tools::ConfigFile.new
+          config << file_or_hash
+          return nil if config.empty?
+          workflow = self.new
+          workflow.configure(config.to_hash.key_symbols_to_strings(recursive: true))
+          workflow
+        end
+
+      end
+    end
+  end
+end

data/libis-workflow-activerecord.gemspec

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'libis/workflow/activerecord/version'
+require 'date'
+
+Gem::Specification.new do |spec|
+  spec.name = 'libis-workflow-activerecord'
+  spec.version = ::Libis::Workflow::ActiveRecord::VERSION
+  spec.date = Date.today.to_s
+
+  spec.summary = %q{ActiveRecord persistence for the LIBIS Workflow framework.}
+  spec.description = %q{Class implementations that use ActiveRecord storage for the LIBIS Workflow framework.}
+
+  spec.author = 'Kris Dekeyser'
+  spec.email = 'kris.dekeyser@libis.be'
+  spec.homepage = 'https://github.com/libis/workflow-activerecord'
+  spec.license = 'MIT'
+
+  spec.platform = Gem::Platform::JAVA if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
+
+  spec.files = `git ls-files -z`.split("\0")
+  spec.executables = spec.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
+
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'libis-workflow', '~> 2.0'
+  spec.add_runtime_dependency 'activerecord', '~> 5'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'awesome_print'
+  spec.add_development_dependency 'database_cleaner'
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'sqlite3'
+  spec.add_runtime_dependency 'pg'
+
+end

data/spec/db_env.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+export DB_HOST="kahana.mongohq.com:10053"
+export DB_HOST="localhost:27017"
+export DB_USER="test"
+export DB_PASS="abc123"

data/spec/db_start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+cd `dirname $0`
+mkdir -p ./data/db
+/opt/mongodb/bin/mongod --dbpath ./data/db
+

data/spec/items.rb

@@ -0,0 +1,3 @@
+require_relative 'items/test_dir_item'
+require_relative 'items/test_file_item'
+require_relative 'items/test_run'

data/spec/items/test_dir_item.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require 'libis/workflow'
+require_relative 'test_item'
+
+class TestDirItem < TestItem
+
+  include ::Libis::Workflow::Base::DirItem
+
+  def name=(dir)
+    raise RuntimeError, "'#{dir}' is not a directory" unless File.directory? dir
+    super dir
+  end
+
+  def name
+    self.properties['name'] || super
+  end
+
+end

data/spec/items/test_file_item.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require 'libis-tools'
+require 'libis-workflow'
+
+require_relative 'test_item'
+
+class TestFileItem < TestItem
+
+  include ::Libis::Workflow::Base::FileItem
+
+  def filename=(file)
+    raise RuntimeError, "'#{file}' is not a file" unless File.file? file
+    set_checksum :SHA256, ::Libis::Tools::Checksum.hexdigest(file, :SHA256)
+    super file
+  end
+
+  def name
+    self.properties['name'] || super
+  end
+
+  def filesize
+    properties['size']
+  end
+
+  def fixity_check(checksum)
+    properties['checksum'] == checksum
+  end
+
+end

data/spec/items/test_item.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+
+require 'libis/workflow/activerecord/work_item'
+
+class TestItem < Libis::Workflow::ActiveRecord::WorkItem
+end

data/spec/items/test_run.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require 'libis-workflow-activerecord'
+
+class TestRun <::Libis::Workflow::ActiveRecord::Run
+
+  def name; 'TestRun'; end
+
+end

data/spec/job_spec.rb

@@ -0,0 +1,10 @@
+require 'rspec'
+require_relative 'spec_helper'
+
+require 'libis-workflow-activerecord'
+require_relative 'test_job'
+
+describe 'TestJob' do
+
+  context
+end

data/spec/run_spec.rb

@@ -0,0 +1,4 @@
+require 'rspec'
+require_relative 'spec_helper'
+
+require 'libis-workflow-activerecord'

data/spec/spec_helper.rb

@@ -0,0 +1,37 @@
+require 'coveralls'
+Coveralls.wear!
+
+require 'bundler/setup'
+Bundler.setup
+
+require 'rspec'
+require 'libis-workflow-activerecord'
+
+require 'database_cleaner'
+require 'stringio'
+
+RSpec.configure do |cfg|
+
+  cfg.before :suite do
+    # noinspection RubyResolve
+    ::Libis::Workflow::ActiveRecord.configure do |cfg|
+      cfg.logger.appenders =
+          ::Logging::Appenders.string_io('StringIO', layout: ::Libis::Tools::Config.get_log_formatter)
+      cfg.itemdir = File.join(File.dirname(__FILE__), 'items')
+      cfg.taskdir = File.join(File.dirname(__FILE__), 'tasks')
+      cfg.workdir = File.join(File.dirname(__FILE__), 'work')
+      cfg.database_connect 'db/config.yml', :test
+    end
+    DatabaseCleaner.clean_with :truncation
+  end
+
+  cfg.before :each do
+    # DatabaseCleaner.strategy = :transaction
+    # DatabaseCleaner.start
+  end
+
+  cfg.after :each do
+    # DatabaseCleaner.clean
+  end
+
+end

data/spec/tasks/camelize_name.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require 'backports/rails/string'
+require 'libis-workflow'
+
+class CamelizeName < ::Libis::Workflow::Task
+
+  def process(item)
+    return unless (item.is_a?(TestFileItem) || item.is_a?(TestDirItem))
+    item.properties['name'] = item.name.camelize
+    item.save!
+  end
+
+end

data/spec/tasks/checksum_tester.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+require 'libis/tools/checksum'
+
+require 'libis/exceptions'
+require 'libis-workflow'
+
+class ChecksumTester < ::Libis::Workflow::Task
+
+  parameter checksum_type: nil,
+            description: 'Checksum type to use.',
+            constraint: ::Libis::Tools::Checksum::CHECKSUM_TYPES.map {|x| x.to_s}
+  parameter checksum_file: nil, description: 'File with checksums of the files.'
+
+  def process(item)
+    return unless item.is_a? TestFileItem
+
+    checksum_type = parameter(:checksum_type)
+
+    if checksum_type.nil?
+      ::Libis::Tools::Checksum::CHECKSUM_TYPES.each do |x|
+        test_checksum(item, x) if item.checksum(x)
+      end
+    else
+      test_checksum(item, checksum_type)
+    end
+  end
+
+  def test_checksum(item, checksum_type)
+    checksum = ::Libis::Tools::Checksum.hexdigest(item.fullpath, checksum_type.to_sym)
+    return if item.checksum(checksum_type) == checksum
+    raise ::Libis::WorkflowError, "Checksum test #{checksum_type} failed for #{item.filepath}"
+  end
+
+end

data/spec/tasks/collect_files.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+require 'libis/exceptions'
+
+require_relative '../items'
+
+class CollectFiles < ::Libis::Workflow::Task
+
+  parameter location: '.',
+            description: 'Dir location to start scanning for files.'
+  parameter subdirs: false,
+            description: 'Look for files in subdirs too.'
+  parameter selection: nil,
+            description: 'Only select files that match the given regular expression. Ignored if empty.'
+
+  def process(item)
+    if item.is_a? TestRun
+      add_item(item, parameter(:location))
+    elsif item.is_a? TestDirItem
+      collect_files(item, item.fullpath)
+    end
+  end
+
+  def collect_files(item, dir)
+    glob_string = dir
+    glob_string = File.join(glob_string, '**') if parameter(:subdirs)
+    glob_string = File.join(glob_string, '*')
+
+    Dir.glob(glob_string).select do |x|
+      parameter(:selection) && !parameter(:selection).empty? ? x =~ Regexp.new(parameter(:selection)) : true
+    end.sort.each do |file|
+      next if %w'. ..'.include? file
+      add_item(item, file)
+    end
+  end
+
+  def add_item(item, file)
+    child = if File.file?(file)
+              TestFileItem.new
+            elsif File.directory?(file)
+              TestDirItem.new
+            else
+              error 'Bad file type encountered: %s', file
+              nil
+            end
+    unless child
+      return
+    end
+    child.filename = file
+    item << child
+  end
+
+end

data/spec/test_job.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+
+require 'libis/workflow/activerecord/job'
+
+class TestJob < ::Libis::Workflow::ActiveRecord::Job
+end

data/spec/test_workflow.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+
+require 'libis/workflow/activerecord/workflow'
+
+class TestWorkflow < ::Libis::Workflow::ActiveRecord::Workflow
+end

data/spec/test_workitem.rb

@@ -0,0 +1,7 @@
+require 'libis/workflow/activerecord/work_item'
+
+class TestWorkItem < ::Libis::Workflow::ActiveRecord::WorkItem
+  property_field :abc,
+                 reader: lambda {|value| value.blank? ? :xyz : value.to_sym},
+                 writer: lambda {|value| value.blank? ? nil : value.to_s}
+end

data/spec/workflow_spec.rb

@@ -0,0 +1,208 @@
+# encoding: utf-8
+
+require 'rspec'
+require 'stringio'
+
+require 'libis-workflow-activerecord'
+
+require_relative 'spec_helper'
+require_relative 'test_job'
+require_relative 'test_workflow'
+require_relative 'items'
+
+require 'awesome_print'
+
+describe 'TestWorkflow' do
+
+  before :each do
+    ::Libis::Workflow::ActiveRecord::Config.logger.appenders.first.clear
+  end
+
+  dirname = File.absolute_path(File.join(File.dirname(__FILE__), 'items'))
+
+  let(:logoutput) { ::Libis::Workflow::ActiveRecord::Config.logger.appenders.first.sio }
+
+  let(:workflow) {
+    wf = TestWorkflow.find_or_initialize_by(name: 'TestWorkflow')
+    wf.configure(
+        'name' => 'TestWorkflow',
+        'description' => 'Workflow for testing',
+        'tasks' => [
+            {'class' => 'CollectFiles', 'recursive' => true},
+            {
+                'name' => 'ProcessFiles',
+                'subitems' => true,
+                'tasks' => [
+                    {'class' => 'ChecksumTester', 'recursive' => true},
+                    {'class' => 'CamelizeName', 'recursive' => true}
+                ]
+            }
+        ],
+        'input' => {
+            'dirname' => {'default' => '.', 'propagate_to' => 'CollectFiles#location'},
+            'checksum_type' => {'default' => 'SHA1', 'propagate_to' => 'ProcessFiles/ChecksumTester'}
+        }
+    )
+    wf.save!
+    wf
+  }
+  let(:job) {
+    job = TestJob.from_hash(
+        'name' => 'TestJob',
+        'description' => 'Job for testing',
+        'workflow' => 'TestWorkflow',
+        'run_object' => 'TestRun',
+        'input' => {'dirname' => dirname, 'checksum_type' => 'SHA256'},
+        'log_to_file' => false,
+        'log_each_run' => false
+    )
+
+    # job = TestJob.find_or_initialize_by(name: 'TestJob')
+    # job.configure(
+    #     'name' => 'TestJob',
+    #     'description' => 'Job for testing',
+    #     'workflow' => workflow,
+    #     'run_object' => 'TestRun',
+    #     'input' => {'dirname' => dirname, 'checksum_type' => 'SHA256'},
+    #     'log_to_file' => false
+    # )
+
+    # noinspection RubyResolve
+    job.runs.each { |run| run.destroy! }
+    job.save!
+    job
+  }
+
+  let(:run) { job.execute }
+
+  it 'should contain three tasks' do
+
+    expect(workflow.config['tasks'].size).to eq 2
+    expect(workflow.config['tasks'].first['class']).to eq 'CollectFiles'
+    expect(workflow.config['tasks'].last['name']).to eq 'ProcessFiles'
+
+  end
+
+  it 'find workflow' do
+    workflow
+    wf = TestWorkflow.first
+    expect(wf.nil?).to eq false
+    expect(wf.name).to eq 'TestWorkflow'
+    expect(wf.description).to eq 'Workflow for testing'
+    expect(wf.input.count).to eq 2
+    expect(wf.input[:dirname][:default]).to eq '.'
+    expect(wf.config['tasks'].count).to eq 2
+    expect(wf.config['tasks'][0]['class']).to eq 'CollectFiles'
+    expect(wf.config['tasks'][0]['recursive']).to eq true
+    expect(wf.config['tasks'][1]['name']).to eq 'ProcessFiles'
+    expect(wf.config['tasks'][1]['subitems']).to eq true
+    expect(wf.config['tasks'][1]['tasks'].count).to eq 2
+    expect(wf.config['tasks'][1]['tasks'][0]['class']).to eq 'ChecksumTester'
+    expect(wf.config['tasks'][1]['tasks'][0]['recursive']).to eq true
+    expect(wf.config['tasks'][1]['tasks'][1]['class']).to eq 'CamelizeName'
+    expect(wf.config['tasks'][1]['tasks'][1]['recursive']).to eq true
+  end
+
+  it 'should camelize the workitem name' do
+
+    expect(run.options['CollectFiles']['location']).to eq dirname
+
+    expect(run.items.size).to eq 1
+    expect(run.items.count).to eq 1
+    expect(run.size).to eq 1
+    expect(run.count).to eq 1
+    expect(run.items.first.class).to eq TestDirItem
+    expect(run.first.class).to eq TestDirItem
+    expect(run.first.items.size).to eq 4
+    expect(run.first.items.count).to eq 4
+    expect(run.first.size).to eq 4
+    expect(run.first.count).to eq 4
+    expect(run.first.items.first.class).to eq TestFileItem
+    expect(run.first.first.class).to eq TestFileItem
+
+    run.items.first.each_with_index do |x, i|
+      expect(x.name).to eq %w'TestDirItem.rb TestFileItem.rb TestItem.rb TestRun.rb'[i]
+    end
+  end
+
+  it 'should return expected debug output' do
+
+    run
+
+    sample_out = <<STR
+Run - TestRun : Ingest run started.
+Run - TestRun : Running subtask (1/2): CollectFiles
+CollectFiles - TestRun : Processing subitem (1/1): items
+CollectFiles - items : Processing subitem (1/4): test_dir_item.rb
+CollectFiles - items : Processing subitem (2/4): test_file_item.rb
+CollectFiles - items : Processing subitem (3/4): test_item.rb
+CollectFiles - items : Processing subitem (4/4): test_run.rb
+CollectFiles - items : 4 of 4 subitems passed
+CollectFiles - TestRun : 1 of 1 subitems passed
+Run - TestRun : Running subtask (2/2): ProcessFiles
+ProcessFiles - TestRun : Running subtask (1/2): ChecksumTester
+ProcessFiles/ChecksumTester - TestRun : Processing subitem (1/1): items
+ProcessFiles/ChecksumTester - items : Processing subitem (1/4): test_dir_item.rb
+ProcessFiles/ChecksumTester - items : Processing subitem (2/4): test_file_item.rb
+ProcessFiles/ChecksumTester - items : Processing subitem (3/4): test_item.rb
+ProcessFiles/ChecksumTester - items : Processing subitem (4/4): test_run.rb
+ProcessFiles/ChecksumTester - items : 4 of 4 subitems passed
+ProcessFiles/ChecksumTester - TestRun : 1 of 1 subitems passed
+ProcessFiles - TestRun : Running subtask (2/2): CamelizeName
+ProcessFiles/CamelizeName - TestRun : Processing subitem (1/1): items
+ProcessFiles/CamelizeName - Items : Processing subitem (1/4): test_dir_item.rb
+ProcessFiles/CamelizeName - Items : Processing subitem (2/4): test_file_item.rb
+ProcessFiles/CamelizeName - Items : Processing subitem (3/4): test_item.rb
+ProcessFiles/CamelizeName - Items : Processing subitem (4/4): test_run.rb
+ProcessFiles/CamelizeName - Items : 4 of 4 subitems passed
+ProcessFiles/CamelizeName - TestRun : 1 of 1 subitems passed
+ProcessFiles - TestRun : Done
+Run - TestRun : Done
+STR
+
+    sample_out = sample_out.lines.to_a
+    output = logoutput.string.lines
+
+    # puts output
+
+    expect(run.status_log.count).to eq 5
+    item = run.items.first
+    expect(item.status_log.count).to eq 3
+
+    expect(output.count).to eq sample_out.count
+    output.each_with_index do |o, i|
+      expect(o.strip).to match(/#{Regexp.escape sample_out[i].strip}$/)
+    end
+
+  end
+
+  # noinspection RubyResolve
+  it 'find run' do
+    run
+    my_job = TestJob.first
+    expect(my_job).to eq job
+    expect(my_job.runs.all.count).to eq 1
+    my_run = my_job.runs.all.first
+    expect(my_run).to eq run
+    expect(my_run.status).to eq :DONE
+  end
+
+  # noinspection RubyResolve
+  it 'find first item' do
+    item = run.items.first
+    expect(item.nil?).to eq false
+    expect(item.is_a? TestDirItem).to eq true
+    expect(item.properties['name']).to eq 'Items'
+  end
+
+  it 'move item in relation' do
+    item = run.items.first
+    sub_item = item.items.first
+    expect(run.size).to eq 1
+    expect(item.size).to eq 4
+    run.move_item(sub_item)
+    expect(run.size).to eq 2
+    expect(item.size).to eq 3
+  end
+
+end