libis-workflow 2.0.25 → 2.0.28

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.coveralls.yml +1 -1
  3. data/.gitignore +36 -36
  4. data/.travis.yml +32 -32
  5. data/Gemfile +4 -4
  6. data/LICENSE +20 -20
  7. data/README.md +380 -380
  8. data/Rakefile +6 -6
  9. data/lib/libis/exceptions.rb +6 -6
  10. data/lib/libis/workflow.rb +41 -41
  11. data/lib/libis/workflow/action.rb +24 -24
  12. data/lib/libis/workflow/base/dir_item.rb +13 -13
  13. data/lib/libis/workflow/base/file_item.rb +80 -80
  14. data/lib/libis/workflow/base/job.rb +83 -83
  15. data/lib/libis/workflow/base/logging.rb +66 -66
  16. data/lib/libis/workflow/base/run.rb +97 -95
  17. data/lib/libis/workflow/base/work_item.rb +173 -173
  18. data/lib/libis/workflow/base/workflow.rb +149 -149
  19. data/lib/libis/workflow/config.rb +22 -22
  20. data/lib/libis/workflow/dir_item.rb +10 -10
  21. data/lib/libis/workflow/file_item.rb +15 -15
  22. data/lib/libis/workflow/job.rb +28 -28
  23. data/lib/libis/workflow/message_registry.rb +30 -30
  24. data/lib/libis/workflow/run.rb +34 -34
  25. data/lib/libis/workflow/status.rb +133 -133
  26. data/lib/libis/workflow/task.rb +318 -316
  27. data/lib/libis/workflow/task_group.rb +72 -71
  28. data/lib/libis/workflow/task_runner.rb +34 -34
  29. data/lib/libis/workflow/version.rb +5 -5
  30. data/lib/libis/workflow/work_item.rb +37 -37
  31. data/lib/libis/workflow/worker.rb +42 -42
  32. data/lib/libis/workflow/workflow.rb +20 -20
  33. data/libis-workflow.gemspec +38 -38
  34. data/spec/items.rb +2 -2
  35. data/spec/items/test_dir_item.rb +13 -13
  36. data/spec/items/test_file_item.rb +16 -16
  37. data/spec/items/test_run.rb +8 -8
  38. data/spec/spec_helper.rb +8 -8
  39. data/spec/task_spec.rb +15 -15
  40. data/spec/tasks/camelize_name.rb +12 -12
  41. data/spec/tasks/checksum_tester.rb +32 -32
  42. data/spec/tasks/collect_files.rb +47 -47
  43. data/spec/workflow_spec.rb +154 -154
  44. metadata +3 -3
@@ -1,71 +1,72 @@
1
- require_relative 'task'
2
-
3
- module Libis
4
- module Workflow
5
-
6
- # noinspection RubyTooManyMethodsInspection
7
- class TaskGroup < Libis::Workflow::Task
8
-
9
- parameter abort_on_failure: true,
10
- description: 'Stop processing tasks if one task fails.'
11
-
12
- attr_accessor :tasks
13
-
14
- def initialize(parent, cfg = {})
15
- self.tasks = []
16
- super parent, cfg
17
- end
18
-
19
- def <<(task)
20
- self.tasks << task
21
- task.parent = self
22
- end
23
-
24
- def apply_options(opts)
25
- super opts
26
- self.tasks.each do |task|
27
- task.apply_options opts
28
- end
29
- end
30
-
31
- protected
32
-
33
- def process(item)
34
-
35
- return unless check_processing_subtasks
36
-
37
- tasks = subtasks
38
- return unless tasks.size > 0
39
-
40
- status = Hash.new(0)
41
- item.status_progress(self.namepath, 0, tasks.count)
42
- tasks.each_with_index do |task, i|
43
- info 'Running subtask (%d/%d): %s', item, i+1, tasks.size, task.name
44
- item = task.run item
45
- item.status_progress(self.namepath, i+1)
46
- item_status = item.status(task.namepath)
47
- status[item_status] += 1
48
- break if parameter(:abort_on_failure) && item_status != :DONE
49
- end
50
-
51
- substatus_check(status, item, 'task')
52
-
53
- info item.status_text(self.namepath).capitalize, item
54
- end
55
-
56
- def stop_processing_subtasks
57
- @subtasks_stopper= true
58
- end
59
-
60
- def check_processing_subtasks
61
- if @subtasks_stopper
62
- @subtasks_stopper = false
63
- return false
64
- end
65
- true
66
- end
67
-
68
- end
69
-
70
- end
71
- end
1
+ require_relative 'task'
2
+
3
+ module Libis
4
+ module Workflow
5
+
6
+ # noinspection RubyTooManyMethodsInspection
7
+ class TaskGroup < Libis::Workflow::Task
8
+
9
+ parameter abort_on_failure: true,
10
+ description: 'Stop processing tasks if one task fails.'
11
+
12
+ attr_accessor :tasks
13
+
14
+ def initialize(parent, cfg = {})
15
+ self.tasks = []
16
+ super parent, cfg
17
+ end
18
+
19
+ def <<(task)
20
+ self.tasks << task
21
+ task.parent = self
22
+ end
23
+
24
+ def apply_options(opts)
25
+ super opts
26
+ self.tasks.each do |task|
27
+ task.apply_options opts
28
+ end
29
+ end
30
+
31
+ protected
32
+
33
+ def process(item)
34
+
35
+ return unless check_processing_subtasks
36
+
37
+ tasks = subtasks
38
+ return unless tasks.size > 0
39
+
40
+ status = Hash.new(0)
41
+ item.status_progress(self.namepath, 0, tasks.count)
42
+ tasks.each_with_index do |task, i|
43
+ info 'Running subtask (%d/%d): %s', item, i+1, tasks.size, task.name
44
+ new_item = task.run item
45
+ item = new_item if new_item.is_a?(Libis::Workflow::WorkItem)
46
+ item.status_progress(self.namepath, i+1)
47
+ item_status = item.status(task.namepath)
48
+ status[item_status] += 1
49
+ break if parameter(:abort_on_failure) && item_status != :DONE
50
+ end
51
+
52
+ substatus_check(status, item, 'task')
53
+
54
+ info item.status_text(self.namepath).capitalize, item
55
+ end
56
+
57
+ def stop_processing_subtasks
58
+ @subtasks_stopper= true
59
+ end
60
+
61
+ def check_processing_subtasks
62
+ if @subtasks_stopper
63
+ @subtasks_stopper = false
64
+ return false
65
+ end
66
+ true
67
+ end
68
+
69
+ end
70
+
71
+ end
72
+ end
@@ -1,34 +1,34 @@
1
- require_relative 'task_group'
2
-
3
- module Libis
4
- module Workflow
5
-
6
- # noinspection RubyTooManyMethodsInspection
7
- class TaskRunner < Libis::Workflow::TaskGroup
8
-
9
- parameter abort_on_failure: true, frozen: true
10
- parameter recursive: false, frozen: true
11
-
12
- def names
13
- Array.new
14
- end
15
-
16
- def namepath
17
- 'Run'
18
- end
19
-
20
- def run(item)
21
-
22
- check_item_type ::Libis::Workflow::Base::WorkItem, item
23
- self.workitem = item
24
-
25
- info 'Ingest run started.', item
26
-
27
- super
28
-
29
- end
30
-
31
- end
32
-
33
- end
34
- end
1
+ require_relative 'task_group'
2
+
3
+ module Libis
4
+ module Workflow
5
+
6
+ # noinspection RubyTooManyMethodsInspection
7
+ class TaskRunner < Libis::Workflow::TaskGroup
8
+
9
+ parameter abort_on_failure: true, frozen: true
10
+ parameter recursive: false, frozen: true
11
+
12
+ def names
13
+ Array.new
14
+ end
15
+
16
+ def namepath
17
+ 'Run'
18
+ end
19
+
20
+ def run(item)
21
+
22
+ check_item_type ::Libis::Workflow::Base::WorkItem, item
23
+ self.workitem = item
24
+
25
+ info 'Ingest run started.', item
26
+
27
+ super
28
+
29
+ end
30
+
31
+ end
32
+
33
+ end
34
+ end
@@ -1,5 +1,5 @@
1
- module Libis
2
- module Workflow
3
- VERSION = '2.0.25' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
4
- end
5
- end
1
+ module Libis
2
+ module Workflow
3
+ VERSION = '2.0.28' unless const_defined? :VERSION # the guard is against a redefinition warning that happens on Travis
4
+ end
5
+ end
@@ -1,37 +1,37 @@
1
- require 'libis/tools/extend/hash'
2
- require 'libis/workflow/base/work_item'
3
-
4
- module Libis
5
- module Workflow
6
-
7
- # In-memory implementation of ::Libis::Workflow::Base::WorkItem
8
- class WorkItem
9
- include ::Libis::Workflow::Base::WorkItem
10
-
11
- attr_accessor :parent
12
- attr_accessor :items
13
- attr_accessor :options, :properties
14
- attr_accessor :status_log
15
- attr_accessor :summary
16
-
17
- def initialize
18
- self.parent = nil
19
- self.items = []
20
- self.options = {}
21
- self.properties = {}
22
- self.status_log = []
23
- self.summary = {}
24
- end
25
-
26
- protected
27
-
28
- def add_status_log(info)
29
- # noinspection RubyResolve
30
- self.status_log << info
31
- self.status_log.last
32
- end
33
-
34
- end
35
-
36
- end
37
- end
1
+ require 'libis/tools/extend/hash'
2
+ require 'libis/workflow/base/work_item'
3
+
4
+ module Libis
5
+ module Workflow
6
+
7
+ # In-memory implementation of ::Libis::Workflow::Base::WorkItem
8
+ class WorkItem
9
+ include ::Libis::Workflow::Base::WorkItem
10
+
11
+ attr_accessor :parent
12
+ attr_accessor :items
13
+ attr_accessor :options, :properties
14
+ attr_accessor :status_log
15
+ attr_accessor :summary
16
+
17
+ def initialize
18
+ self.parent = nil
19
+ self.items = []
20
+ self.options = {}
21
+ self.properties = {}
22
+ self.status_log = []
23
+ self.summary = {}
24
+ end
25
+
26
+ protected
27
+
28
+ def add_status_log(info)
29
+ # noinspection RubyResolve
30
+ self.status_log << info
31
+ self.status_log.last
32
+ end
33
+
34
+ end
35
+
36
+ end
37
+ end
@@ -1,42 +1,42 @@
1
- require 'sidekiq'
2
-
3
- require 'libis/tools/extend/hash'
4
- require 'libis/workflow/config'
5
- require 'libis/workflow/workflow'
6
-
7
- module Libis
8
- module Workflow
9
-
10
- class Worker
11
- include Sidekiq::Worker
12
-
13
- def perform(job_config, options = {})
14
- job = configure(job_config, options)
15
- options[:interactive] = false
16
- job.execute options
17
- end
18
-
19
- def configure(job_config, options = {})
20
- log_path = options.delete :log_path
21
- if log_path
22
- Libis::Workflow::Config.logger = ::Logger.new(
23
- File.join(log_path, "#{job_config[:name]}.log"),
24
- (options.delete(:log_shift_age) || 'daily'),
25
- (options.delete(:log_shift_size) || 1024 ** 2)
26
- )
27
- Libis::Workflow::Config.logger.formatter = ::Logger::Formatter.new
28
- Libis::Workflow::Config.logger.level = (options.delete(:log_level) || ::Logger::DEBUG)
29
- end
30
- get_job(job_config)
31
- end
32
-
33
- def get_job(job_config)
34
- job = ::Libis::Workflow::Job.new
35
- job.configure job_config.key_symbols_to_strings(recursive: true)
36
- job
37
- end
38
-
39
- end
40
-
41
- end
42
- end
1
+ require 'sidekiq'
2
+
3
+ require 'libis/tools/extend/hash'
4
+ require 'libis/workflow/config'
5
+ require 'libis/workflow/workflow'
6
+
7
+ module Libis
8
+ module Workflow
9
+
10
+ class Worker
11
+ include Sidekiq::Worker
12
+
13
+ def perform(job_config, options = {})
14
+ job = configure(job_config, options)
15
+ options[:interactive] = false
16
+ job.execute options
17
+ end
18
+
19
+ def configure(job_config, options = {})
20
+ log_path = options.delete :log_path
21
+ if log_path
22
+ Libis::Workflow::Config.logger = ::Logger.new(
23
+ File.join(log_path, "#{job_config[:name]}.log"),
24
+ (options.delete(:log_shift_age) || 'daily'),
25
+ (options.delete(:log_shift_size) || 1024 ** 2)
26
+ )
27
+ Libis::Workflow::Config.logger.formatter = ::Logger::Formatter.new
28
+ Libis::Workflow::Config.logger.level = (options.delete(:log_level) || ::Logger::DEBUG)
29
+ end
30
+ get_job(job_config)
31
+ end
32
+
33
+ def get_job(job_config)
34
+ job = ::Libis::Workflow::Job.new
35
+ job.configure job_config.key_symbols_to_strings(recursive: true)
36
+ job
37
+ end
38
+
39
+ end
40
+
41
+ end
42
+ end
@@ -1,20 +1,20 @@
1
- require 'libis/workflow/base/workflow'
2
-
3
- module Libis
4
- module Workflow
5
-
6
- class Workflow
7
- include ::Libis::Workflow::Base::Workflow
8
-
9
- attr_accessor :name, :description, :config
10
-
11
- def initialize
12
- @name = ''
13
- @description = ''
14
- @config = Hash.new
15
- end
16
-
17
- end
18
-
19
- end
20
- end
1
+ require 'libis/workflow/base/workflow'
2
+
3
+ module Libis
4
+ module Workflow
5
+
6
+ class Workflow
7
+ include ::Libis::Workflow::Base::Workflow
8
+
9
+ attr_accessor :name, :description, :config
10
+
11
+ def initialize
12
+ @name = ''
13
+ @description = ''
14
+ @config = Hash.new
15
+ end
16
+
17
+ end
18
+
19
+ end
20
+ end
@@ -1,38 +1,38 @@
1
- # encoding: utf-8
2
-
3
- lib = File.expand_path('../lib', __FILE__)
4
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
-
6
- require 'libis/workflow/version'
7
-
8
- Gem::Specification.new do |spec|
9
- spec.name = 'libis-workflow'
10
- spec.version = ::Libis::Workflow::VERSION
11
- spec.date = Date.today.to_s
12
-
13
- spec.summary = %q{LIBIS Workflow framework.}
14
- spec.description = %q{A simple framework to build custom task/workflow solutions.}
15
-
16
- spec.author = 'Kris Dekeyser'
17
- spec.email = 'kris.dekeyser@libis.be'
18
- spec.homepage = 'https://github.com/Kris-LIBIS/workflow'
19
- spec.license = 'MIT'
20
-
21
- spec.platform = Gem::Platform::JAVA if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
22
-
23
- spec.files = `git ls-files -z`.split("\x0")
24
- spec.executables = spec.files.grep(%r{^bin/}).map { |f| File.basename(f) }
25
- spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
26
-
27
- spec.require_paths = ['lib']
28
-
29
- spec.add_development_dependency 'bundler', '~> 1.6'
30
- spec.add_development_dependency 'rake', '~> 10.3'
31
- spec.add_development_dependency 'rspec', '~> 3.1'
32
- spec.add_development_dependency 'simplecov', '~> 0.9'
33
- spec.add_development_dependency 'coveralls', '~> 0.7'
34
-
35
- spec.add_runtime_dependency 'libis-tools', '~> 0.9'
36
- spec.add_runtime_dependency 'sidekiq', '~> 4.0'
37
- spec.add_runtime_dependency 'eldritch', '~> 1.1'
38
- end
1
+ # encoding: utf-8
2
+
3
+ lib = File.expand_path('../lib', __FILE__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+
6
+ require 'libis/workflow/version'
7
+
8
+ Gem::Specification.new do |spec|
9
+ spec.name = 'libis-workflow'
10
+ spec.version = ::Libis::Workflow::VERSION
11
+ spec.date = Date.today.to_s
12
+
13
+ spec.summary = %q{LIBIS Workflow framework.}
14
+ spec.description = %q{A simple framework to build custom task/workflow solutions.}
15
+
16
+ spec.author = 'Kris Dekeyser'
17
+ spec.email = 'kris.dekeyser@libis.be'
18
+ spec.homepage = 'https://github.com/Kris-LIBIS/workflow'
19
+ spec.license = 'MIT'
20
+
21
+ spec.platform = Gem::Platform::JAVA if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
22
+
23
+ spec.files = `git ls-files -z`.split("\x0")
24
+ spec.executables = spec.files.grep(%r{^bin/}).map { |f| File.basename(f) }
25
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
26
+
27
+ spec.require_paths = ['lib']
28
+
29
+ spec.add_development_dependency 'bundler', '~> 1.6'
30
+ spec.add_development_dependency 'rake', '~> 10.3'
31
+ spec.add_development_dependency 'rspec', '~> 3.1'
32
+ spec.add_development_dependency 'simplecov', '~> 0.9'
33
+ spec.add_development_dependency 'coveralls', '~> 0.7'
34
+
35
+ spec.add_runtime_dependency 'libis-tools', '~> 0.9'
36
+ spec.add_runtime_dependency 'sidekiq', '~> 4.0'
37
+ spec.add_runtime_dependency 'eldritch', '~> 1.1'
38
+ end