libis-workflow 2.0.25 → 2.0.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.coveralls.yml +1 -1
- data/.gitignore +36 -36
- data/.travis.yml +32 -32
- data/Gemfile +4 -4
- data/LICENSE +20 -20
- data/README.md +380 -380
- data/Rakefile +6 -6
- data/lib/libis/exceptions.rb +6 -6
- data/lib/libis/workflow.rb +41 -41
- data/lib/libis/workflow/action.rb +24 -24
- data/lib/libis/workflow/base/dir_item.rb +13 -13
- data/lib/libis/workflow/base/file_item.rb +80 -80
- data/lib/libis/workflow/base/job.rb +83 -83
- data/lib/libis/workflow/base/logging.rb +66 -66
- data/lib/libis/workflow/base/run.rb +97 -95
- data/lib/libis/workflow/base/work_item.rb +173 -173
- data/lib/libis/workflow/base/workflow.rb +149 -149
- data/lib/libis/workflow/config.rb +22 -22
- data/lib/libis/workflow/dir_item.rb +10 -10
- data/lib/libis/workflow/file_item.rb +15 -15
- data/lib/libis/workflow/job.rb +28 -28
- data/lib/libis/workflow/message_registry.rb +30 -30
- data/lib/libis/workflow/run.rb +34 -34
- data/lib/libis/workflow/status.rb +133 -133
- data/lib/libis/workflow/task.rb +318 -316
- data/lib/libis/workflow/task_group.rb +72 -71
- data/lib/libis/workflow/task_runner.rb +34 -34
- data/lib/libis/workflow/version.rb +5 -5
- data/lib/libis/workflow/work_item.rb +37 -37
- data/lib/libis/workflow/worker.rb +42 -42
- data/lib/libis/workflow/workflow.rb +20 -20
- data/libis-workflow.gemspec +38 -38
- data/spec/items.rb +2 -2
- data/spec/items/test_dir_item.rb +13 -13
- data/spec/items/test_file_item.rb +16 -16
- data/spec/items/test_run.rb +8 -8
- data/spec/spec_helper.rb +8 -8
- data/spec/task_spec.rb +15 -15
- data/spec/tasks/camelize_name.rb +12 -12
- data/spec/tasks/checksum_tester.rb +32 -32
- data/spec/tasks/collect_files.rb +47 -47
- data/spec/workflow_spec.rb +154 -154
- metadata +3 -3
@@ -1,71 +1,72 @@
|
|
1
|
-
require_relative 'task'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
|
6
|
-
# noinspection RubyTooManyMethodsInspection
|
7
|
-
class TaskGroup < Libis::Workflow::Task
|
8
|
-
|
9
|
-
parameter abort_on_failure: true,
|
10
|
-
description: 'Stop processing tasks if one task fails.'
|
11
|
-
|
12
|
-
attr_accessor :tasks
|
13
|
-
|
14
|
-
def initialize(parent, cfg = {})
|
15
|
-
self.tasks = []
|
16
|
-
super parent, cfg
|
17
|
-
end
|
18
|
-
|
19
|
-
def <<(task)
|
20
|
-
self.tasks << task
|
21
|
-
task.parent = self
|
22
|
-
end
|
23
|
-
|
24
|
-
def apply_options(opts)
|
25
|
-
super opts
|
26
|
-
self.tasks.each do |task|
|
27
|
-
task.apply_options opts
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
protected
|
32
|
-
|
33
|
-
def process(item)
|
34
|
-
|
35
|
-
return unless check_processing_subtasks
|
36
|
-
|
37
|
-
tasks = subtasks
|
38
|
-
return unless tasks.size > 0
|
39
|
-
|
40
|
-
status = Hash.new(0)
|
41
|
-
item.status_progress(self.namepath, 0, tasks.count)
|
42
|
-
tasks.each_with_index do |task, i|
|
43
|
-
info 'Running subtask (%d/%d): %s', item, i+1, tasks.size, task.name
|
44
|
-
|
45
|
-
item.
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
end
|
1
|
+
require_relative 'task'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
|
6
|
+
# noinspection RubyTooManyMethodsInspection
|
7
|
+
class TaskGroup < Libis::Workflow::Task
|
8
|
+
|
9
|
+
parameter abort_on_failure: true,
|
10
|
+
description: 'Stop processing tasks if one task fails.'
|
11
|
+
|
12
|
+
attr_accessor :tasks
|
13
|
+
|
14
|
+
def initialize(parent, cfg = {})
|
15
|
+
self.tasks = []
|
16
|
+
super parent, cfg
|
17
|
+
end
|
18
|
+
|
19
|
+
def <<(task)
|
20
|
+
self.tasks << task
|
21
|
+
task.parent = self
|
22
|
+
end
|
23
|
+
|
24
|
+
def apply_options(opts)
|
25
|
+
super opts
|
26
|
+
self.tasks.each do |task|
|
27
|
+
task.apply_options opts
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def process(item)
|
34
|
+
|
35
|
+
return unless check_processing_subtasks
|
36
|
+
|
37
|
+
tasks = subtasks
|
38
|
+
return unless tasks.size > 0
|
39
|
+
|
40
|
+
status = Hash.new(0)
|
41
|
+
item.status_progress(self.namepath, 0, tasks.count)
|
42
|
+
tasks.each_with_index do |task, i|
|
43
|
+
info 'Running subtask (%d/%d): %s', item, i+1, tasks.size, task.name
|
44
|
+
new_item = task.run item
|
45
|
+
item = new_item if new_item.is_a?(Libis::Workflow::WorkItem)
|
46
|
+
item.status_progress(self.namepath, i+1)
|
47
|
+
item_status = item.status(task.namepath)
|
48
|
+
status[item_status] += 1
|
49
|
+
break if parameter(:abort_on_failure) && item_status != :DONE
|
50
|
+
end
|
51
|
+
|
52
|
+
substatus_check(status, item, 'task')
|
53
|
+
|
54
|
+
info item.status_text(self.namepath).capitalize, item
|
55
|
+
end
|
56
|
+
|
57
|
+
def stop_processing_subtasks
|
58
|
+
@subtasks_stopper= true
|
59
|
+
end
|
60
|
+
|
61
|
+
def check_processing_subtasks
|
62
|
+
if @subtasks_stopper
|
63
|
+
@subtasks_stopper = false
|
64
|
+
return false
|
65
|
+
end
|
66
|
+
true
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
@@ -1,34 +1,34 @@
|
|
1
|
-
require_relative 'task_group'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
|
6
|
-
# noinspection RubyTooManyMethodsInspection
|
7
|
-
class TaskRunner < Libis::Workflow::TaskGroup
|
8
|
-
|
9
|
-
parameter abort_on_failure: true, frozen: true
|
10
|
-
parameter recursive: false, frozen: true
|
11
|
-
|
12
|
-
def names
|
13
|
-
Array.new
|
14
|
-
end
|
15
|
-
|
16
|
-
def namepath
|
17
|
-
'Run'
|
18
|
-
end
|
19
|
-
|
20
|
-
def run(item)
|
21
|
-
|
22
|
-
check_item_type ::Libis::Workflow::Base::WorkItem, item
|
23
|
-
self.workitem = item
|
24
|
-
|
25
|
-
info 'Ingest run started.', item
|
26
|
-
|
27
|
-
super
|
28
|
-
|
29
|
-
end
|
30
|
-
|
31
|
-
end
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
1
|
+
require_relative 'task_group'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
|
6
|
+
# noinspection RubyTooManyMethodsInspection
|
7
|
+
class TaskRunner < Libis::Workflow::TaskGroup
|
8
|
+
|
9
|
+
parameter abort_on_failure: true, frozen: true
|
10
|
+
parameter recursive: false, frozen: true
|
11
|
+
|
12
|
+
def names
|
13
|
+
Array.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def namepath
|
17
|
+
'Run'
|
18
|
+
end
|
19
|
+
|
20
|
+
def run(item)
|
21
|
+
|
22
|
+
check_item_type ::Libis::Workflow::Base::WorkItem, item
|
23
|
+
self.workitem = item
|
24
|
+
|
25
|
+
info 'Ingest run started.', item
|
26
|
+
|
27
|
+
super
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
module Libis
|
2
|
-
module Workflow
|
3
|
-
VERSION = '2.0.
|
4
|
-
end
|
5
|
-
end
|
1
|
+
module Libis
|
2
|
+
module Workflow
|
3
|
+
VERSION = '2.0.28' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
|
4
|
+
end
|
5
|
+
end
|
@@ -1,37 +1,37 @@
|
|
1
|
-
require 'libis/tools/extend/hash'
|
2
|
-
require 'libis/workflow/base/work_item'
|
3
|
-
|
4
|
-
module Libis
|
5
|
-
module Workflow
|
6
|
-
|
7
|
-
# In-memory implementation of ::Libis::Workflow::Base::WorkItem
|
8
|
-
class WorkItem
|
9
|
-
include ::Libis::Workflow::Base::WorkItem
|
10
|
-
|
11
|
-
attr_accessor :parent
|
12
|
-
attr_accessor :items
|
13
|
-
attr_accessor :options, :properties
|
14
|
-
attr_accessor :status_log
|
15
|
-
attr_accessor :summary
|
16
|
-
|
17
|
-
def initialize
|
18
|
-
self.parent = nil
|
19
|
-
self.items = []
|
20
|
-
self.options = {}
|
21
|
-
self.properties = {}
|
22
|
-
self.status_log = []
|
23
|
-
self.summary = {}
|
24
|
-
end
|
25
|
-
|
26
|
-
protected
|
27
|
-
|
28
|
-
def add_status_log(info)
|
29
|
-
# noinspection RubyResolve
|
30
|
-
self.status_log << info
|
31
|
-
self.status_log.last
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
end
|
37
|
-
end
|
1
|
+
require 'libis/tools/extend/hash'
|
2
|
+
require 'libis/workflow/base/work_item'
|
3
|
+
|
4
|
+
module Libis
|
5
|
+
module Workflow
|
6
|
+
|
7
|
+
# In-memory implementation of ::Libis::Workflow::Base::WorkItem
|
8
|
+
class WorkItem
|
9
|
+
include ::Libis::Workflow::Base::WorkItem
|
10
|
+
|
11
|
+
attr_accessor :parent
|
12
|
+
attr_accessor :items
|
13
|
+
attr_accessor :options, :properties
|
14
|
+
attr_accessor :status_log
|
15
|
+
attr_accessor :summary
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
self.parent = nil
|
19
|
+
self.items = []
|
20
|
+
self.options = {}
|
21
|
+
self.properties = {}
|
22
|
+
self.status_log = []
|
23
|
+
self.summary = {}
|
24
|
+
end
|
25
|
+
|
26
|
+
protected
|
27
|
+
|
28
|
+
def add_status_log(info)
|
29
|
+
# noinspection RubyResolve
|
30
|
+
self.status_log << info
|
31
|
+
self.status_log.last
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
end
|
@@ -1,42 +1,42 @@
|
|
1
|
-
require 'sidekiq'
|
2
|
-
|
3
|
-
require 'libis/tools/extend/hash'
|
4
|
-
require 'libis/workflow/config'
|
5
|
-
require 'libis/workflow/workflow'
|
6
|
-
|
7
|
-
module Libis
|
8
|
-
module Workflow
|
9
|
-
|
10
|
-
class Worker
|
11
|
-
include Sidekiq::Worker
|
12
|
-
|
13
|
-
def perform(job_config, options = {})
|
14
|
-
job = configure(job_config, options)
|
15
|
-
options[:interactive] = false
|
16
|
-
job.execute options
|
17
|
-
end
|
18
|
-
|
19
|
-
def configure(job_config, options = {})
|
20
|
-
log_path = options.delete :log_path
|
21
|
-
if log_path
|
22
|
-
Libis::Workflow::Config.logger = ::Logger.new(
|
23
|
-
File.join(log_path, "#{job_config[:name]}.log"),
|
24
|
-
(options.delete(:log_shift_age) || 'daily'),
|
25
|
-
(options.delete(:log_shift_size) || 1024 ** 2)
|
26
|
-
)
|
27
|
-
Libis::Workflow::Config.logger.formatter = ::Logger::Formatter.new
|
28
|
-
Libis::Workflow::Config.logger.level = (options.delete(:log_level) || ::Logger::DEBUG)
|
29
|
-
end
|
30
|
-
get_job(job_config)
|
31
|
-
end
|
32
|
-
|
33
|
-
def get_job(job_config)
|
34
|
-
job = ::Libis::Workflow::Job.new
|
35
|
-
job.configure job_config.key_symbols_to_strings(recursive: true)
|
36
|
-
job
|
37
|
-
end
|
38
|
-
|
39
|
-
end
|
40
|
-
|
41
|
-
end
|
42
|
-
end
|
1
|
+
require 'sidekiq'
|
2
|
+
|
3
|
+
require 'libis/tools/extend/hash'
|
4
|
+
require 'libis/workflow/config'
|
5
|
+
require 'libis/workflow/workflow'
|
6
|
+
|
7
|
+
module Libis
|
8
|
+
module Workflow
|
9
|
+
|
10
|
+
class Worker
|
11
|
+
include Sidekiq::Worker
|
12
|
+
|
13
|
+
def perform(job_config, options = {})
|
14
|
+
job = configure(job_config, options)
|
15
|
+
options[:interactive] = false
|
16
|
+
job.execute options
|
17
|
+
end
|
18
|
+
|
19
|
+
def configure(job_config, options = {})
|
20
|
+
log_path = options.delete :log_path
|
21
|
+
if log_path
|
22
|
+
Libis::Workflow::Config.logger = ::Logger.new(
|
23
|
+
File.join(log_path, "#{job_config[:name]}.log"),
|
24
|
+
(options.delete(:log_shift_age) || 'daily'),
|
25
|
+
(options.delete(:log_shift_size) || 1024 ** 2)
|
26
|
+
)
|
27
|
+
Libis::Workflow::Config.logger.formatter = ::Logger::Formatter.new
|
28
|
+
Libis::Workflow::Config.logger.level = (options.delete(:log_level) || ::Logger::DEBUG)
|
29
|
+
end
|
30
|
+
get_job(job_config)
|
31
|
+
end
|
32
|
+
|
33
|
+
def get_job(job_config)
|
34
|
+
job = ::Libis::Workflow::Job.new
|
35
|
+
job.configure job_config.key_symbols_to_strings(recursive: true)
|
36
|
+
job
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
@@ -1,20 +1,20 @@
|
|
1
|
-
require 'libis/workflow/base/workflow'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
|
6
|
-
class Workflow
|
7
|
-
include ::Libis::Workflow::Base::Workflow
|
8
|
-
|
9
|
-
attr_accessor :name, :description, :config
|
10
|
-
|
11
|
-
def initialize
|
12
|
-
@name = ''
|
13
|
-
@description = ''
|
14
|
-
@config = Hash.new
|
15
|
-
end
|
16
|
-
|
17
|
-
end
|
18
|
-
|
19
|
-
end
|
20
|
-
end
|
1
|
+
require 'libis/workflow/base/workflow'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
|
6
|
+
class Workflow
|
7
|
+
include ::Libis::Workflow::Base::Workflow
|
8
|
+
|
9
|
+
attr_accessor :name, :description, :config
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@name = ''
|
13
|
+
@description = ''
|
14
|
+
@config = Hash.new
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
end
|
data/libis-workflow.gemspec
CHANGED
@@ -1,38 +1,38 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
|
-
lib = File.expand_path('../lib', __FILE__)
|
4
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
-
|
6
|
-
require 'libis/workflow/version'
|
7
|
-
|
8
|
-
Gem::Specification.new do |spec|
|
9
|
-
spec.name = 'libis-workflow'
|
10
|
-
spec.version = ::Libis::Workflow::VERSION
|
11
|
-
spec.date = Date.today.to_s
|
12
|
-
|
13
|
-
spec.summary = %q{LIBIS Workflow framework.}
|
14
|
-
spec.description = %q{A simple framework to build custom task/workflow solutions.}
|
15
|
-
|
16
|
-
spec.author = 'Kris Dekeyser'
|
17
|
-
spec.email = 'kris.dekeyser@libis.be'
|
18
|
-
spec.homepage = 'https://github.com/Kris-LIBIS/workflow'
|
19
|
-
spec.license = 'MIT'
|
20
|
-
|
21
|
-
spec.platform = Gem::Platform::JAVA if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
|
22
|
-
|
23
|
-
spec.files = `git ls-files -z`.split("\x0")
|
24
|
-
spec.executables = spec.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
25
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
26
|
-
|
27
|
-
spec.require_paths = ['lib']
|
28
|
-
|
29
|
-
spec.add_development_dependency 'bundler', '~> 1.6'
|
30
|
-
spec.add_development_dependency 'rake', '~> 10.3'
|
31
|
-
spec.add_development_dependency 'rspec', '~> 3.1'
|
32
|
-
spec.add_development_dependency 'simplecov', '~> 0.9'
|
33
|
-
spec.add_development_dependency 'coveralls', '~> 0.7'
|
34
|
-
|
35
|
-
spec.add_runtime_dependency 'libis-tools', '~> 0.9'
|
36
|
-
spec.add_runtime_dependency 'sidekiq', '~> 4.0'
|
37
|
-
spec.add_runtime_dependency 'eldritch', '~> 1.1'
|
38
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
|
6
|
+
require 'libis/workflow/version'
|
7
|
+
|
8
|
+
Gem::Specification.new do |spec|
|
9
|
+
spec.name = 'libis-workflow'
|
10
|
+
spec.version = ::Libis::Workflow::VERSION
|
11
|
+
spec.date = Date.today.to_s
|
12
|
+
|
13
|
+
spec.summary = %q{LIBIS Workflow framework.}
|
14
|
+
spec.description = %q{A simple framework to build custom task/workflow solutions.}
|
15
|
+
|
16
|
+
spec.author = 'Kris Dekeyser'
|
17
|
+
spec.email = 'kris.dekeyser@libis.be'
|
18
|
+
spec.homepage = 'https://github.com/Kris-LIBIS/workflow'
|
19
|
+
spec.license = 'MIT'
|
20
|
+
|
21
|
+
spec.platform = Gem::Platform::JAVA if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
|
22
|
+
|
23
|
+
spec.files = `git ls-files -z`.split("\x0")
|
24
|
+
spec.executables = spec.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
25
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
26
|
+
|
27
|
+
spec.require_paths = ['lib']
|
28
|
+
|
29
|
+
spec.add_development_dependency 'bundler', '~> 1.6'
|
30
|
+
spec.add_development_dependency 'rake', '~> 10.3'
|
31
|
+
spec.add_development_dependency 'rspec', '~> 3.1'
|
32
|
+
spec.add_development_dependency 'simplecov', '~> 0.9'
|
33
|
+
spec.add_development_dependency 'coveralls', '~> 0.7'
|
34
|
+
|
35
|
+
spec.add_runtime_dependency 'libis-tools', '~> 0.9'
|
36
|
+
spec.add_runtime_dependency 'sidekiq', '~> 4.0'
|
37
|
+
spec.add_runtime_dependency 'eldritch', '~> 1.1'
|
38
|
+
end
|