libis-workflow 2.0.24 → 2.0.25
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.coveralls.yml +1 -1
- data/.gitignore +36 -36
- data/.travis.yml +32 -32
- data/Gemfile +4 -4
- data/LICENSE +20 -20
- data/README.md +380 -380
- data/Rakefile +6 -6
- data/lib/libis/exceptions.rb +6 -6
- data/lib/libis/workflow.rb +41 -41
- data/lib/libis/workflow/action.rb +24 -24
- data/lib/libis/workflow/base/dir_item.rb +13 -13
- data/lib/libis/workflow/base/file_item.rb +80 -80
- data/lib/libis/workflow/base/job.rb +83 -83
- data/lib/libis/workflow/base/logging.rb +66 -66
- data/lib/libis/workflow/base/run.rb +95 -95
- data/lib/libis/workflow/base/work_item.rb +173 -173
- data/lib/libis/workflow/base/workflow.rb +149 -149
- data/lib/libis/workflow/config.rb +22 -22
- data/lib/libis/workflow/dir_item.rb +10 -10
- data/lib/libis/workflow/file_item.rb +15 -15
- data/lib/libis/workflow/job.rb +28 -28
- data/lib/libis/workflow/message_registry.rb +30 -30
- data/lib/libis/workflow/run.rb +34 -34
- data/lib/libis/workflow/status.rb +133 -133
- data/lib/libis/workflow/task.rb +316 -316
- data/lib/libis/workflow/task_group.rb +71 -71
- data/lib/libis/workflow/task_runner.rb +34 -34
- data/lib/libis/workflow/version.rb +5 -5
- data/lib/libis/workflow/work_item.rb +37 -37
- data/lib/libis/workflow/worker.rb +42 -42
- data/lib/libis/workflow/workflow.rb +20 -20
- data/libis-workflow.gemspec +38 -38
- data/spec/items.rb +2 -2
- data/spec/items/test_dir_item.rb +13 -13
- data/spec/items/test_file_item.rb +16 -16
- data/spec/items/test_run.rb +8 -8
- data/spec/spec_helper.rb +8 -8
- data/spec/task_spec.rb +15 -15
- data/spec/tasks/camelize_name.rb +12 -12
- data/spec/tasks/checksum_tester.rb +32 -32
- data/spec/tasks/collect_files.rb +47 -47
- data/spec/workflow_spec.rb +154 -154
- metadata +3 -3
@@ -1,71 +1,71 @@
|
|
1
|
-
require_relative 'task'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
|
6
|
-
# noinspection RubyTooManyMethodsInspection
|
7
|
-
class TaskGroup < Libis::Workflow::Task
|
8
|
-
|
9
|
-
parameter abort_on_failure: true,
|
10
|
-
description: 'Stop processing tasks if one task fails.'
|
11
|
-
|
12
|
-
attr_accessor :tasks
|
13
|
-
|
14
|
-
def initialize(parent, cfg = {})
|
15
|
-
self.tasks = []
|
16
|
-
super parent, cfg
|
17
|
-
end
|
18
|
-
|
19
|
-
def <<(task)
|
20
|
-
self.tasks << task
|
21
|
-
task.parent = self
|
22
|
-
end
|
23
|
-
|
24
|
-
def apply_options(opts)
|
25
|
-
super opts
|
26
|
-
self.tasks.each do |task|
|
27
|
-
task.apply_options opts
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
protected
|
32
|
-
|
33
|
-
def process(item)
|
34
|
-
|
35
|
-
return unless check_processing_subtasks
|
36
|
-
|
37
|
-
tasks = subtasks
|
38
|
-
return unless tasks.size > 0
|
39
|
-
|
40
|
-
status = Hash.new(0)
|
41
|
-
item.status_progress(self.namepath, 0, tasks.count)
|
42
|
-
tasks.each_with_index do |task, i|
|
43
|
-
info 'Running subtask (%d/%d): %s', item, i+1, tasks.size, task.name
|
44
|
-
item = task.run item
|
45
|
-
item.status_progress(self.namepath, i+1)
|
46
|
-
item_status = item.status(task.namepath)
|
47
|
-
status[item_status] += 1
|
48
|
-
break if parameter(:abort_on_failure) && item_status != :DONE
|
49
|
-
end
|
50
|
-
|
51
|
-
substatus_check(status, item, 'task')
|
52
|
-
|
53
|
-
info item.status_text(self.namepath).capitalize, item
|
54
|
-
end
|
55
|
-
|
56
|
-
def stop_processing_subtasks
|
57
|
-
@subtasks_stopper= true
|
58
|
-
end
|
59
|
-
|
60
|
-
def check_processing_subtasks
|
61
|
-
if @subtasks_stopper
|
62
|
-
@subtasks_stopper = false
|
63
|
-
return false
|
64
|
-
end
|
65
|
-
true
|
66
|
-
end
|
67
|
-
|
68
|
-
end
|
69
|
-
|
70
|
-
end
|
71
|
-
end
|
1
|
+
require_relative 'task'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
|
6
|
+
# noinspection RubyTooManyMethodsInspection
|
7
|
+
class TaskGroup < Libis::Workflow::Task
|
8
|
+
|
9
|
+
parameter abort_on_failure: true,
|
10
|
+
description: 'Stop processing tasks if one task fails.'
|
11
|
+
|
12
|
+
attr_accessor :tasks
|
13
|
+
|
14
|
+
def initialize(parent, cfg = {})
|
15
|
+
self.tasks = []
|
16
|
+
super parent, cfg
|
17
|
+
end
|
18
|
+
|
19
|
+
def <<(task)
|
20
|
+
self.tasks << task
|
21
|
+
task.parent = self
|
22
|
+
end
|
23
|
+
|
24
|
+
def apply_options(opts)
|
25
|
+
super opts
|
26
|
+
self.tasks.each do |task|
|
27
|
+
task.apply_options opts
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def process(item)
|
34
|
+
|
35
|
+
return unless check_processing_subtasks
|
36
|
+
|
37
|
+
tasks = subtasks
|
38
|
+
return unless tasks.size > 0
|
39
|
+
|
40
|
+
status = Hash.new(0)
|
41
|
+
item.status_progress(self.namepath, 0, tasks.count)
|
42
|
+
tasks.each_with_index do |task, i|
|
43
|
+
info 'Running subtask (%d/%d): %s', item, i+1, tasks.size, task.name
|
44
|
+
item = task.run item
|
45
|
+
item.status_progress(self.namepath, i+1)
|
46
|
+
item_status = item.status(task.namepath)
|
47
|
+
status[item_status] += 1
|
48
|
+
break if parameter(:abort_on_failure) && item_status != :DONE
|
49
|
+
end
|
50
|
+
|
51
|
+
substatus_check(status, item, 'task')
|
52
|
+
|
53
|
+
info item.status_text(self.namepath).capitalize, item
|
54
|
+
end
|
55
|
+
|
56
|
+
def stop_processing_subtasks
|
57
|
+
@subtasks_stopper= true
|
58
|
+
end
|
59
|
+
|
60
|
+
def check_processing_subtasks
|
61
|
+
if @subtasks_stopper
|
62
|
+
@subtasks_stopper = false
|
63
|
+
return false
|
64
|
+
end
|
65
|
+
true
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
end
|
@@ -1,34 +1,34 @@
|
|
1
|
-
require_relative 'task_group'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
|
6
|
-
# noinspection RubyTooManyMethodsInspection
|
7
|
-
class TaskRunner < Libis::Workflow::TaskGroup
|
8
|
-
|
9
|
-
parameter abort_on_failure: true, frozen: true
|
10
|
-
parameter recursive: false, frozen: true
|
11
|
-
|
12
|
-
def names
|
13
|
-
Array.new
|
14
|
-
end
|
15
|
-
|
16
|
-
def namepath
|
17
|
-
'Run'
|
18
|
-
end
|
19
|
-
|
20
|
-
def run(item)
|
21
|
-
|
22
|
-
check_item_type ::Libis::Workflow::Base::WorkItem, item
|
23
|
-
self.workitem = item
|
24
|
-
|
25
|
-
info 'Ingest run started.', item
|
26
|
-
|
27
|
-
super
|
28
|
-
|
29
|
-
end
|
30
|
-
|
31
|
-
end
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
1
|
+
require_relative 'task_group'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
|
6
|
+
# noinspection RubyTooManyMethodsInspection
|
7
|
+
class TaskRunner < Libis::Workflow::TaskGroup
|
8
|
+
|
9
|
+
parameter abort_on_failure: true, frozen: true
|
10
|
+
parameter recursive: false, frozen: true
|
11
|
+
|
12
|
+
def names
|
13
|
+
Array.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def namepath
|
17
|
+
'Run'
|
18
|
+
end
|
19
|
+
|
20
|
+
def run(item)
|
21
|
+
|
22
|
+
check_item_type ::Libis::Workflow::Base::WorkItem, item
|
23
|
+
self.workitem = item
|
24
|
+
|
25
|
+
info 'Ingest run started.', item
|
26
|
+
|
27
|
+
super
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
module Libis
|
2
|
-
module Workflow
|
3
|
-
VERSION = '2.0.
|
4
|
-
end
|
5
|
-
end
|
1
|
+
module Libis
|
2
|
+
module Workflow
|
3
|
+
VERSION = '2.0.25' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
|
4
|
+
end
|
5
|
+
end
|
@@ -1,37 +1,37 @@
|
|
1
|
-
require 'libis/tools/extend/hash'
|
2
|
-
require 'libis/workflow/base/work_item'
|
3
|
-
|
4
|
-
module Libis
|
5
|
-
module Workflow
|
6
|
-
|
7
|
-
# In-memory implementation of ::Libis::Workflow::Base::WorkItem
|
8
|
-
class WorkItem
|
9
|
-
include ::Libis::Workflow::Base::WorkItem
|
10
|
-
|
11
|
-
attr_accessor :parent
|
12
|
-
attr_accessor :items
|
13
|
-
attr_accessor :options, :properties
|
14
|
-
attr_accessor :status_log
|
15
|
-
attr_accessor :summary
|
16
|
-
|
17
|
-
def initialize
|
18
|
-
self.parent = nil
|
19
|
-
self.items = []
|
20
|
-
self.options = {}
|
21
|
-
self.properties = {}
|
22
|
-
self.status_log = []
|
23
|
-
self.summary = {}
|
24
|
-
end
|
25
|
-
|
26
|
-
protected
|
27
|
-
|
28
|
-
def add_status_log(info)
|
29
|
-
# noinspection RubyResolve
|
30
|
-
self.status_log << info
|
31
|
-
self.status_log.last
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
end
|
37
|
-
end
|
1
|
+
require 'libis/tools/extend/hash'
|
2
|
+
require 'libis/workflow/base/work_item'
|
3
|
+
|
4
|
+
module Libis
|
5
|
+
module Workflow
|
6
|
+
|
7
|
+
# In-memory implementation of ::Libis::Workflow::Base::WorkItem
|
8
|
+
class WorkItem
|
9
|
+
include ::Libis::Workflow::Base::WorkItem
|
10
|
+
|
11
|
+
attr_accessor :parent
|
12
|
+
attr_accessor :items
|
13
|
+
attr_accessor :options, :properties
|
14
|
+
attr_accessor :status_log
|
15
|
+
attr_accessor :summary
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
self.parent = nil
|
19
|
+
self.items = []
|
20
|
+
self.options = {}
|
21
|
+
self.properties = {}
|
22
|
+
self.status_log = []
|
23
|
+
self.summary = {}
|
24
|
+
end
|
25
|
+
|
26
|
+
protected
|
27
|
+
|
28
|
+
def add_status_log(info)
|
29
|
+
# noinspection RubyResolve
|
30
|
+
self.status_log << info
|
31
|
+
self.status_log.last
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
end
|
@@ -1,42 +1,42 @@
|
|
1
|
-
require 'sidekiq'
|
2
|
-
|
3
|
-
require 'libis/tools/extend/hash'
|
4
|
-
require 'libis/workflow/config'
|
5
|
-
require 'libis/workflow/workflow'
|
6
|
-
|
7
|
-
module Libis
|
8
|
-
module Workflow
|
9
|
-
|
10
|
-
class Worker
|
11
|
-
include Sidekiq::Worker
|
12
|
-
|
13
|
-
def perform(job_config, options = {})
|
14
|
-
job = configure(job_config, options)
|
15
|
-
options[:interactive] = false
|
16
|
-
job.execute options
|
17
|
-
end
|
18
|
-
|
19
|
-
def configure(job_config, options = {})
|
20
|
-
log_path = options.delete :log_path
|
21
|
-
if log_path
|
22
|
-
Libis::Workflow::Config.logger = ::Logger.new(
|
23
|
-
File.join(log_path, "#{job_config[:name]}.log"),
|
24
|
-
(options.delete(:log_shift_age) || 'daily'),
|
25
|
-
(options.delete(:log_shift_size) || 1024 ** 2)
|
26
|
-
)
|
27
|
-
Libis::Workflow::Config.logger.formatter = ::Logger::Formatter.new
|
28
|
-
Libis::Workflow::Config.logger.level = (options.delete(:log_level) || ::Logger::DEBUG)
|
29
|
-
end
|
30
|
-
get_job(job_config)
|
31
|
-
end
|
32
|
-
|
33
|
-
def get_job(job_config)
|
34
|
-
job = ::Libis::Workflow::Job.new
|
35
|
-
job.configure job_config.key_symbols_to_strings(recursive: true)
|
36
|
-
job
|
37
|
-
end
|
38
|
-
|
39
|
-
end
|
40
|
-
|
41
|
-
end
|
42
|
-
end
|
1
|
+
require 'sidekiq'
|
2
|
+
|
3
|
+
require 'libis/tools/extend/hash'
|
4
|
+
require 'libis/workflow/config'
|
5
|
+
require 'libis/workflow/workflow'
|
6
|
+
|
7
|
+
module Libis
|
8
|
+
module Workflow
|
9
|
+
|
10
|
+
class Worker
|
11
|
+
include Sidekiq::Worker
|
12
|
+
|
13
|
+
def perform(job_config, options = {})
|
14
|
+
job = configure(job_config, options)
|
15
|
+
options[:interactive] = false
|
16
|
+
job.execute options
|
17
|
+
end
|
18
|
+
|
19
|
+
def configure(job_config, options = {})
|
20
|
+
log_path = options.delete :log_path
|
21
|
+
if log_path
|
22
|
+
Libis::Workflow::Config.logger = ::Logger.new(
|
23
|
+
File.join(log_path, "#{job_config[:name]}.log"),
|
24
|
+
(options.delete(:log_shift_age) || 'daily'),
|
25
|
+
(options.delete(:log_shift_size) || 1024 ** 2)
|
26
|
+
)
|
27
|
+
Libis::Workflow::Config.logger.formatter = ::Logger::Formatter.new
|
28
|
+
Libis::Workflow::Config.logger.level = (options.delete(:log_level) || ::Logger::DEBUG)
|
29
|
+
end
|
30
|
+
get_job(job_config)
|
31
|
+
end
|
32
|
+
|
33
|
+
def get_job(job_config)
|
34
|
+
job = ::Libis::Workflow::Job.new
|
35
|
+
job.configure job_config.key_symbols_to_strings(recursive: true)
|
36
|
+
job
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
@@ -1,20 +1,20 @@
|
|
1
|
-
require 'libis/workflow/base/workflow'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
|
6
|
-
class Workflow
|
7
|
-
include ::Libis::Workflow::Base::Workflow
|
8
|
-
|
9
|
-
attr_accessor :name, :description, :config
|
10
|
-
|
11
|
-
def initialize
|
12
|
-
@name = ''
|
13
|
-
@description = ''
|
14
|
-
@config = Hash.new
|
15
|
-
end
|
16
|
-
|
17
|
-
end
|
18
|
-
|
19
|
-
end
|
20
|
-
end
|
1
|
+
require 'libis/workflow/base/workflow'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
|
6
|
+
class Workflow
|
7
|
+
include ::Libis::Workflow::Base::Workflow
|
8
|
+
|
9
|
+
attr_accessor :name, :description, :config
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@name = ''
|
13
|
+
@description = ''
|
14
|
+
@config = Hash.new
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
end
|
data/libis-workflow.gemspec
CHANGED
@@ -1,38 +1,38 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
|
-
lib = File.expand_path('../lib', __FILE__)
|
4
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
-
|
6
|
-
require 'libis/workflow/version'
|
7
|
-
|
8
|
-
Gem::Specification.new do |spec|
|
9
|
-
spec.name = 'libis-workflow'
|
10
|
-
spec.version = ::Libis::Workflow::VERSION
|
11
|
-
spec.date = Date.today.to_s
|
12
|
-
|
13
|
-
spec.summary = %q{LIBIS Workflow framework.}
|
14
|
-
spec.description = %q{A simple framework to build custom task/workflow solutions.}
|
15
|
-
|
16
|
-
spec.author = 'Kris Dekeyser'
|
17
|
-
spec.email = 'kris.dekeyser@libis.be'
|
18
|
-
spec.homepage = 'https://github.com/Kris-LIBIS/workflow'
|
19
|
-
spec.license = 'MIT'
|
20
|
-
|
21
|
-
spec.platform = Gem::Platform::JAVA if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
|
22
|
-
|
23
|
-
spec.files = `git ls-files -z`.split("\x0")
|
24
|
-
spec.executables = spec.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
25
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
26
|
-
|
27
|
-
spec.require_paths = ['lib']
|
28
|
-
|
29
|
-
spec.add_development_dependency 'bundler', '~> 1.6'
|
30
|
-
spec.add_development_dependency 'rake', '~> 10.3'
|
31
|
-
spec.add_development_dependency 'rspec', '~> 3.1'
|
32
|
-
spec.add_development_dependency 'simplecov', '~> 0.9'
|
33
|
-
spec.add_development_dependency 'coveralls', '~> 0.7'
|
34
|
-
|
35
|
-
spec.add_runtime_dependency 'libis-tools', '~> 0.9'
|
36
|
-
spec.add_runtime_dependency 'sidekiq', '~> 4.0'
|
37
|
-
spec.add_runtime_dependency 'eldritch', '~> 1.1'
|
38
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
|
6
|
+
require 'libis/workflow/version'
|
7
|
+
|
8
|
+
Gem::Specification.new do |spec|
|
9
|
+
spec.name = 'libis-workflow'
|
10
|
+
spec.version = ::Libis::Workflow::VERSION
|
11
|
+
spec.date = Date.today.to_s
|
12
|
+
|
13
|
+
spec.summary = %q{LIBIS Workflow framework.}
|
14
|
+
spec.description = %q{A simple framework to build custom task/workflow solutions.}
|
15
|
+
|
16
|
+
spec.author = 'Kris Dekeyser'
|
17
|
+
spec.email = 'kris.dekeyser@libis.be'
|
18
|
+
spec.homepage = 'https://github.com/Kris-LIBIS/workflow'
|
19
|
+
spec.license = 'MIT'
|
20
|
+
|
21
|
+
spec.platform = Gem::Platform::JAVA if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
|
22
|
+
|
23
|
+
spec.files = `git ls-files -z`.split("\x0")
|
24
|
+
spec.executables = spec.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
25
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
26
|
+
|
27
|
+
spec.require_paths = ['lib']
|
28
|
+
|
29
|
+
spec.add_development_dependency 'bundler', '~> 1.6'
|
30
|
+
spec.add_development_dependency 'rake', '~> 10.3'
|
31
|
+
spec.add_development_dependency 'rspec', '~> 3.1'
|
32
|
+
spec.add_development_dependency 'simplecov', '~> 0.9'
|
33
|
+
spec.add_development_dependency 'coveralls', '~> 0.7'
|
34
|
+
|
35
|
+
spec.add_runtime_dependency 'libis-tools', '~> 0.9'
|
36
|
+
spec.add_runtime_dependency 'sidekiq', '~> 4.0'
|
37
|
+
spec.add_runtime_dependency 'eldritch', '~> 1.1'
|
38
|
+
end
|