libis-workflow 2.0.24 → 2.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.coveralls.yml +1 -1
- data/.gitignore +36 -36
- data/.travis.yml +32 -32
- data/Gemfile +4 -4
- data/LICENSE +20 -20
- data/README.md +380 -380
- data/Rakefile +6 -6
- data/lib/libis/exceptions.rb +6 -6
- data/lib/libis/workflow.rb +41 -41
- data/lib/libis/workflow/action.rb +24 -24
- data/lib/libis/workflow/base/dir_item.rb +13 -13
- data/lib/libis/workflow/base/file_item.rb +80 -80
- data/lib/libis/workflow/base/job.rb +83 -83
- data/lib/libis/workflow/base/logging.rb +66 -66
- data/lib/libis/workflow/base/run.rb +95 -95
- data/lib/libis/workflow/base/work_item.rb +173 -173
- data/lib/libis/workflow/base/workflow.rb +149 -149
- data/lib/libis/workflow/config.rb +22 -22
- data/lib/libis/workflow/dir_item.rb +10 -10
- data/lib/libis/workflow/file_item.rb +15 -15
- data/lib/libis/workflow/job.rb +28 -28
- data/lib/libis/workflow/message_registry.rb +30 -30
- data/lib/libis/workflow/run.rb +34 -34
- data/lib/libis/workflow/status.rb +133 -133
- data/lib/libis/workflow/task.rb +316 -316
- data/lib/libis/workflow/task_group.rb +71 -71
- data/lib/libis/workflow/task_runner.rb +34 -34
- data/lib/libis/workflow/version.rb +5 -5
- data/lib/libis/workflow/work_item.rb +37 -37
- data/lib/libis/workflow/worker.rb +42 -42
- data/lib/libis/workflow/workflow.rb +20 -20
- data/libis-workflow.gemspec +38 -38
- data/spec/items.rb +2 -2
- data/spec/items/test_dir_item.rb +13 -13
- data/spec/items/test_file_item.rb +16 -16
- data/spec/items/test_run.rb +8 -8
- data/spec/spec_helper.rb +8 -8
- data/spec/task_spec.rb +15 -15
- data/spec/tasks/camelize_name.rb +12 -12
- data/spec/tasks/checksum_tester.rb +32 -32
- data/spec/tasks/collect_files.rb +47 -47
- data/spec/workflow_spec.rb +154 -154
- metadata +3 -3
@@ -1,30 +1,30 @@
|
|
1
|
-
require 'singleton'
|
2
|
-
|
3
|
-
module Libis
|
4
|
-
module Workflow
|
5
|
-
class MessageRegistry
|
6
|
-
include Singleton
|
7
|
-
|
8
|
-
def initialize
|
9
|
-
@message_db = {}
|
10
|
-
end
|
11
|
-
|
12
|
-
def register_message(id, message)
|
13
|
-
@message_db[id] = message
|
14
|
-
end
|
15
|
-
|
16
|
-
def get_message(id)
|
17
|
-
@message_db[id]
|
18
|
-
end
|
19
|
-
|
20
|
-
def self.register_message(id, message)
|
21
|
-
self.instance.register_message id, message
|
22
|
-
end
|
23
|
-
|
24
|
-
def self.get_message(id)
|
25
|
-
self.instance.get_message id
|
26
|
-
end
|
27
|
-
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
1
|
+
require 'singleton'
|
2
|
+
|
3
|
+
module Libis
|
4
|
+
module Workflow
|
5
|
+
class MessageRegistry
|
6
|
+
include Singleton
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@message_db = {}
|
10
|
+
end
|
11
|
+
|
12
|
+
def register_message(id, message)
|
13
|
+
@message_db[id] = message
|
14
|
+
end
|
15
|
+
|
16
|
+
def get_message(id)
|
17
|
+
@message_db[id]
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.register_message(id, message)
|
21
|
+
self.instance.register_message id, message
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.get_message(id)
|
25
|
+
self.instance.get_message id
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/libis/workflow/run.rb
CHANGED
@@ -1,35 +1,35 @@
|
|
1
|
-
require 'securerandom'
|
2
|
-
|
3
|
-
require 'libis/workflow/config'
|
4
|
-
require 'libis/workflow/workflow'
|
5
|
-
|
6
|
-
require 'libis/workflow/base/run'
|
7
|
-
require 'libis/workflow/work_item'
|
8
|
-
|
9
|
-
module Libis
|
10
|
-
module Workflow
|
11
|
-
|
12
|
-
class Run < ::Libis::Workflow::WorkItem
|
13
|
-
include ::Libis::Workflow::Base::Run
|
14
|
-
|
15
|
-
attr_accessor :start_date, :job, :id
|
16
|
-
|
17
|
-
def initialize
|
18
|
-
@start_date = Time.now
|
19
|
-
@job = nil
|
20
|
-
@id = SecureRandom.hex(10)
|
21
|
-
super
|
22
|
-
end
|
23
|
-
|
24
|
-
def id
|
25
|
-
nil
|
26
|
-
end
|
27
|
-
|
28
|
-
def logger
|
29
|
-
self.properties[:logger] || (job.logger rescue nil)
|
30
|
-
end
|
31
|
-
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
1
|
+
require 'securerandom'
|
2
|
+
|
3
|
+
require 'libis/workflow/config'
|
4
|
+
require 'libis/workflow/workflow'
|
5
|
+
|
6
|
+
require 'libis/workflow/base/run'
|
7
|
+
require 'libis/workflow/work_item'
|
8
|
+
|
9
|
+
module Libis
|
10
|
+
module Workflow
|
11
|
+
|
12
|
+
class Run < ::Libis::Workflow::WorkItem
|
13
|
+
include ::Libis::Workflow::Base::Run
|
14
|
+
|
15
|
+
attr_accessor :start_date, :job, :id
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
@start_date = Time.now
|
19
|
+
@job = nil
|
20
|
+
@id = SecureRandom.hex(10)
|
21
|
+
super
|
22
|
+
end
|
23
|
+
|
24
|
+
def id
|
25
|
+
nil
|
26
|
+
end
|
27
|
+
|
28
|
+
def logger
|
29
|
+
self.properties[:logger] || (job.logger rescue nil)
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
35
|
end
|
@@ -1,133 +1,133 @@
|
|
1
|
-
module Libis
|
2
|
-
module Workflow
|
3
|
-
module Status
|
4
|
-
|
5
|
-
STATUS = {
|
6
|
-
NOT_STARTED: 0,
|
7
|
-
STARTED: 1,
|
8
|
-
DONE: 2,
|
9
|
-
ASYNC_WAIT: 3,
|
10
|
-
ASYNC_HALT: 4,
|
11
|
-
FAILED: 5
|
12
|
-
}
|
13
|
-
|
14
|
-
STATUS_TEXT = [
|
15
|
-
'not started',
|
16
|
-
'started',
|
17
|
-
'done',
|
18
|
-
'waiting for running async process',
|
19
|
-
'waiting for halted async process',
|
20
|
-
'failed'
|
21
|
-
]
|
22
|
-
|
23
|
-
# Changes the status of the object. The status changed is logged in the status_log with the current timestamp.
|
24
|
-
#
|
25
|
-
# @param [String] task namepath of the task
|
26
|
-
# @param [Symbol] status status to set
|
27
|
-
def set_status(task, status)
|
28
|
-
task = task.namepath if task.is_a?(Libis::Workflow::Task)
|
29
|
-
log_entry = self.status_entry(task)
|
30
|
-
case status
|
31
|
-
when :STARTED
|
32
|
-
unless status(task) == :ASYNC_WAIT
|
33
|
-
log_entry = self.add_status_log('task' => task, 'status' => status, 'created' => DateTime.now)
|
34
|
-
end
|
35
|
-
else
|
36
|
-
log_entry ||= self.add_status_log('task' => task, 'status' => status, 'created' => DateTime.now)
|
37
|
-
end
|
38
|
-
log_entry['status'] = status
|
39
|
-
log_entry['updated'] = DateTime.now
|
40
|
-
self.save!
|
41
|
-
end
|
42
|
-
|
43
|
-
# Get last known status symbol for a given task
|
44
|
-
#
|
45
|
-
# @param [String] task task name to check item status for
|
46
|
-
# @return [Symbol] the status code
|
47
|
-
def status(task = nil)
|
48
|
-
entry = self.status_entry(task)
|
49
|
-
status_symbol(entry['status']) rescue :NOT_STARTED
|
50
|
-
end
|
51
|
-
|
52
|
-
# Get last known status text for a given task
|
53
|
-
#
|
54
|
-
# @param [String] task task name to check item status for
|
55
|
-
# @return [Symbol] the status code
|
56
|
-
def status_text(task = nil)
|
57
|
-
entry = self.status_entry(task)
|
58
|
-
status_string(entry['status']) rescue STATUS_TEXT.first
|
59
|
-
end
|
60
|
-
|
61
|
-
# Gets the last known status label of the object.
|
62
|
-
#
|
63
|
-
# @param [String] task name of task to get the status for
|
64
|
-
# @return [String] status label ( = task name + status )
|
65
|
-
def status_label(task = nil)
|
66
|
-
entry = self.status_entry(task)
|
67
|
-
"#{entry['task'] rescue nil}#{entry['status'].capitalize rescue nil}"
|
68
|
-
end
|
69
|
-
|
70
|
-
# Check status of the object.
|
71
|
-
#
|
72
|
-
# @param [Symbol] state status to look for
|
73
|
-
# @param [String] task name of task whose status to check
|
74
|
-
# @return [Boolean] true if the object status matches
|
75
|
-
def check_status(state, task = nil)
|
76
|
-
self.status(task) == state
|
77
|
-
end
|
78
|
-
|
79
|
-
# Compare status with current status of the object.
|
80
|
-
#
|
81
|
-
# @param [Symbol] state
|
82
|
-
# @return [Integer] 1, 0 or -1 depnding on which
|
83
|
-
def compare_status(state, task = nil)
|
84
|
-
STATUS[self.status(task)] <=> STATUS[state]
|
85
|
-
end
|
86
|
-
|
87
|
-
# Update the progress of the working task
|
88
|
-
# @param [String] task namepath of the task
|
89
|
-
# @param [Integer] progress progress indicator (as <progress> of <max> or as % if <max> not set). Default: 0
|
90
|
-
# @param [Integer] max max count.
|
91
|
-
def status_progress(task, progress, max = nil)
|
92
|
-
log_entry = self.status_entry(task)
|
93
|
-
log_entry ||= self.add_status_log('task' => task, 'status' => :STARTED, 'created' => DateTime.now)
|
94
|
-
log_entry['progress'] = progress ? progress : (log_entry['progress'] || 0) + 1
|
95
|
-
log_entry['max'] = max if max
|
96
|
-
log_entry['updated'] = DateTime.now
|
97
|
-
self.save!
|
98
|
-
end
|
99
|
-
|
100
|
-
protected
|
101
|
-
|
102
|
-
# Get last known status entry for a given task
|
103
|
-
#
|
104
|
-
# @param [String] task task name to check item status for
|
105
|
-
# @return [Hash] the status entry
|
106
|
-
def status_entry(task = nil)
|
107
|
-
task = task.namepath if task.is_a?(Libis::Workflow::Task)
|
108
|
-
return self.status_log.last if task.blank?
|
109
|
-
self.status_log.select { |entry| entry['task'] == task }.last
|
110
|
-
end
|
111
|
-
|
112
|
-
# Convert String, Symbol or Integer to correct symbol for the status.
|
113
|
-
# If the input value is nil, the fist status entry is returned.
|
114
|
-
#
|
115
|
-
# @param [String|Symbol|Integer] x string, symbol or integer for status code.
|
116
|
-
# @return [Symbol] the corresponding STATUS symbol
|
117
|
-
def status_symbol(x)
|
118
|
-
return STATUS.key(x) if x.is_a?(Integer)
|
119
|
-
return x if STATUS.has_key?(x)
|
120
|
-
x = x.to_s.upcase.to_sym
|
121
|
-
STATUS.has_key?(x) ? x : nil
|
122
|
-
end
|
123
|
-
|
124
|
-
def status_string(x)
|
125
|
-
return STATUS_TEXT[x] if x.is_a?(Integer)
|
126
|
-
return STATUS_TEXT[STATUS[x]] if STATUS.has_key?(x)
|
127
|
-
x = x.to_s.upcase.to_sym
|
128
|
-
STATUS.has_key?(x) ? STATUS_TEXT[STATUS[x]] : 'unknown status'
|
129
|
-
end
|
130
|
-
|
131
|
-
end
|
132
|
-
end
|
133
|
-
end
|
1
|
+
module Libis
|
2
|
+
module Workflow
|
3
|
+
module Status
|
4
|
+
|
5
|
+
STATUS = {
|
6
|
+
NOT_STARTED: 0,
|
7
|
+
STARTED: 1,
|
8
|
+
DONE: 2,
|
9
|
+
ASYNC_WAIT: 3,
|
10
|
+
ASYNC_HALT: 4,
|
11
|
+
FAILED: 5
|
12
|
+
}
|
13
|
+
|
14
|
+
STATUS_TEXT = [
|
15
|
+
'not started',
|
16
|
+
'started',
|
17
|
+
'done',
|
18
|
+
'waiting for running async process',
|
19
|
+
'waiting for halted async process',
|
20
|
+
'failed'
|
21
|
+
]
|
22
|
+
|
23
|
+
# Changes the status of the object. The status changed is logged in the status_log with the current timestamp.
|
24
|
+
#
|
25
|
+
# @param [String] task namepath of the task
|
26
|
+
# @param [Symbol] status status to set
|
27
|
+
def set_status(task, status)
|
28
|
+
task = task.namepath if task.is_a?(Libis::Workflow::Task)
|
29
|
+
log_entry = self.status_entry(task)
|
30
|
+
case status
|
31
|
+
when :STARTED
|
32
|
+
unless status(task) == :ASYNC_WAIT
|
33
|
+
log_entry = self.add_status_log('task' => task, 'status' => status, 'created' => DateTime.now)
|
34
|
+
end
|
35
|
+
else
|
36
|
+
log_entry ||= self.add_status_log('task' => task, 'status' => status, 'created' => DateTime.now)
|
37
|
+
end
|
38
|
+
log_entry['status'] = status
|
39
|
+
log_entry['updated'] = DateTime.now
|
40
|
+
self.save!
|
41
|
+
end
|
42
|
+
|
43
|
+
# Get last known status symbol for a given task
|
44
|
+
#
|
45
|
+
# @param [String] task task name to check item status for
|
46
|
+
# @return [Symbol] the status code
|
47
|
+
def status(task = nil)
|
48
|
+
entry = self.status_entry(task)
|
49
|
+
status_symbol(entry['status']) rescue :NOT_STARTED
|
50
|
+
end
|
51
|
+
|
52
|
+
# Get last known status text for a given task
|
53
|
+
#
|
54
|
+
# @param [String] task task name to check item status for
|
55
|
+
# @return [Symbol] the status code
|
56
|
+
def status_text(task = nil)
|
57
|
+
entry = self.status_entry(task)
|
58
|
+
status_string(entry['status']) rescue STATUS_TEXT.first
|
59
|
+
end
|
60
|
+
|
61
|
+
# Gets the last known status label of the object.
|
62
|
+
#
|
63
|
+
# @param [String] task name of task to get the status for
|
64
|
+
# @return [String] status label ( = task name + status )
|
65
|
+
def status_label(task = nil)
|
66
|
+
entry = self.status_entry(task)
|
67
|
+
"#{entry['task'] rescue nil}#{entry['status'].capitalize rescue nil}"
|
68
|
+
end
|
69
|
+
|
70
|
+
# Check status of the object.
|
71
|
+
#
|
72
|
+
# @param [Symbol] state status to look for
|
73
|
+
# @param [String] task name of task whose status to check
|
74
|
+
# @return [Boolean] true if the object status matches
|
75
|
+
def check_status(state, task = nil)
|
76
|
+
self.status(task) == state
|
77
|
+
end
|
78
|
+
|
79
|
+
# Compare status with current status of the object.
|
80
|
+
#
|
81
|
+
# @param [Symbol] state
|
82
|
+
# @return [Integer] 1, 0 or -1 depnding on which
|
83
|
+
def compare_status(state, task = nil)
|
84
|
+
STATUS[self.status(task)] <=> STATUS[state]
|
85
|
+
end
|
86
|
+
|
87
|
+
# Update the progress of the working task
|
88
|
+
# @param [String] task namepath of the task
|
89
|
+
# @param [Integer] progress progress indicator (as <progress> of <max> or as % if <max> not set). Default: 0
|
90
|
+
# @param [Integer] max max count.
|
91
|
+
def status_progress(task, progress, max = nil)
|
92
|
+
log_entry = self.status_entry(task)
|
93
|
+
log_entry ||= self.add_status_log('task' => task, 'status' => :STARTED, 'created' => DateTime.now)
|
94
|
+
log_entry['progress'] = progress ? progress : (log_entry['progress'] || 0) + 1
|
95
|
+
log_entry['max'] = max if max
|
96
|
+
log_entry['updated'] = DateTime.now
|
97
|
+
self.save!
|
98
|
+
end
|
99
|
+
|
100
|
+
protected
|
101
|
+
|
102
|
+
# Get last known status entry for a given task
|
103
|
+
#
|
104
|
+
# @param [String] task task name to check item status for
|
105
|
+
# @return [Hash] the status entry
|
106
|
+
def status_entry(task = nil)
|
107
|
+
task = task.namepath if task.is_a?(Libis::Workflow::Task)
|
108
|
+
return self.status_log.last if task.blank?
|
109
|
+
self.status_log.select { |entry| entry['task'] == task }.last
|
110
|
+
end
|
111
|
+
|
112
|
+
# Convert String, Symbol or Integer to correct symbol for the status.
|
113
|
+
# If the input value is nil, the fist status entry is returned.
|
114
|
+
#
|
115
|
+
# @param [String|Symbol|Integer] x string, symbol or integer for status code.
|
116
|
+
# @return [Symbol] the corresponding STATUS symbol
|
117
|
+
def status_symbol(x)
|
118
|
+
return STATUS.key(x) if x.is_a?(Integer)
|
119
|
+
return x if STATUS.has_key?(x)
|
120
|
+
x = x.to_s.upcase.to_sym
|
121
|
+
STATUS.has_key?(x) ? x : nil
|
122
|
+
end
|
123
|
+
|
124
|
+
def status_string(x)
|
125
|
+
return STATUS_TEXT[x] if x.is_a?(Integer)
|
126
|
+
return STATUS_TEXT[STATUS[x]] if STATUS.has_key?(x)
|
127
|
+
x = x.to_s.upcase.to_sym
|
128
|
+
STATUS.has_key?(x) ? STATUS_TEXT[STATUS[x]] : 'unknown status'
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
data/lib/libis/workflow/task.rb
CHANGED
@@ -1,316 +1,316 @@
|
|
1
|
-
require 'backports/rails/hash'
|
2
|
-
require 'backports/rails/string'
|
3
|
-
|
4
|
-
require 'libis/tools/parameter'
|
5
|
-
require 'libis/tools/extend/hash'
|
6
|
-
require 'libis/tools/logger'
|
7
|
-
|
8
|
-
require 'libis/workflow'
|
9
|
-
|
10
|
-
module Libis
|
11
|
-
module Workflow
|
12
|
-
|
13
|
-
# noinspection RubyTooManyMethodsInspection
|
14
|
-
class Task
|
15
|
-
include ::Libis::Tools::Logger
|
16
|
-
include ::Libis::Tools::ParameterContainer
|
17
|
-
|
18
|
-
attr_accessor :parent, :name, :workitem, :processing_item
|
19
|
-
|
20
|
-
parameter recursive: false, description: 'Run the task on all subitems recursively.'
|
21
|
-
parameter abort_recursion_on_failure: false, description: 'Stop processing items recursively if one item fails.'
|
22
|
-
parameter retry_count: 0, description: 'Number of times to retry the task if waiting for another process.'
|
23
|
-
parameter retry_interval: 10, description: 'Number of seconds to wait between retries.'
|
24
|
-
|
25
|
-
def self.task_classes
|
26
|
-
ObjectSpace.each_object(::Class).select { |klass| klass < self }
|
27
|
-
end
|
28
|
-
|
29
|
-
def initialize(parent, cfg = {})
|
30
|
-
@subitems_stopper = false
|
31
|
-
@subtasks_stopper = false
|
32
|
-
self.parent = parent
|
33
|
-
configure cfg
|
34
|
-
end
|
35
|
-
|
36
|
-
def <<(task)
|
37
|
-
raise Libis::WorkflowError, "Processing task '#{self.namepath}' is not allowed to have subtasks."
|
38
|
-
end
|
39
|
-
|
40
|
-
# @param [Libis::Workflow::Base::WorkItem] item
|
41
|
-
def run(item)
|
42
|
-
check_item_type ::Libis::Workflow::Base::WorkItem, item
|
43
|
-
self.workitem = item
|
44
|
-
|
45
|
-
case self.action
|
46
|
-
when :retry
|
47
|
-
if item.check_status(:DONE, self.namepath)
|
48
|
-
debug 'Retry: skipping task %s because it has finished successfully.', item, self.namepath
|
49
|
-
return item
|
50
|
-
end
|
51
|
-
when :failed
|
52
|
-
return item
|
53
|
-
else
|
54
|
-
end
|
55
|
-
|
56
|
-
(parameter(:retry_count)+1).times do
|
57
|
-
|
58
|
-
item = run_item(item)
|
59
|
-
|
60
|
-
case item.status(self.namepath)
|
61
|
-
when :DONE
|
62
|
-
self.action = :run
|
63
|
-
return item
|
64
|
-
when :ASYNC_WAIT
|
65
|
-
self.action = :retry
|
66
|
-
when :ASYNC_HALT
|
67
|
-
break
|
68
|
-
when :FAILED
|
69
|
-
break
|
70
|
-
else
|
71
|
-
return item
|
72
|
-
end
|
73
|
-
|
74
|
-
self.action = :retry
|
75
|
-
|
76
|
-
sleep(parameter(:retry_interval))
|
77
|
-
|
78
|
-
end
|
79
|
-
|
80
|
-
item.get_run.action = :failed
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
item
|
99
|
-
|
100
|
-
end
|
101
|
-
|
102
|
-
def names
|
103
|
-
(self.parent.names rescue Array.new).push(name).compact
|
104
|
-
end
|
105
|
-
|
106
|
-
def namepath;
|
107
|
-
self.names.join('/');
|
108
|
-
end
|
109
|
-
|
110
|
-
def apply_options(opts)
|
111
|
-
o = {}
|
112
|
-
o.merge!(opts[self.class.to_s] || {})
|
113
|
-
o.merge!(opts[self.name] || opts[self.names.join('/')] || {})
|
114
|
-
|
115
|
-
if o and o.is_a? Hash
|
116
|
-
default_values.each do |name, _|
|
117
|
-
next unless o.key?(name.to_s)
|
118
|
-
next if o[name.to_s].nil?
|
119
|
-
paramdef = get_parameter_definition name.to_sym
|
120
|
-
value = paramdef.parse(o[name.to_s])
|
121
|
-
self.parameter(name.to_sym, value)
|
122
|
-
end
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
def message(severity, msg, *args)
|
127
|
-
taskname = self.namepath rescue nil
|
128
|
-
self.set_application(taskname)
|
129
|
-
item = self.workitem rescue nil
|
130
|
-
item = args.shift if args.size > 0 and args[0].is_a?(::Libis::Workflow::Base::WorkItem)
|
131
|
-
subject = item.namepath rescue nil
|
132
|
-
subject ||= item.name rescue nil
|
133
|
-
subject ||= item.to_s rescue nil
|
134
|
-
self.set_subject(subject)
|
135
|
-
super(severity, msg, *args)
|
136
|
-
end
|
137
|
-
|
138
|
-
def logger
|
139
|
-
(self.parent || self.get_run).logger
|
140
|
-
end
|
141
|
-
|
142
|
-
protected
|
143
|
-
|
144
|
-
def configure(cfg)
|
145
|
-
self.name = cfg['name'] || (cfg['class'] || self.class).to_s.split('::').last
|
146
|
-
(cfg['options'] || {}).merge(
|
147
|
-
cfg.reject { |k, _| %w(options name class).include? k }
|
148
|
-
).symbolize_keys.each do |k, v|
|
149
|
-
self.parameter(k, v)
|
150
|
-
end
|
151
|
-
end
|
152
|
-
|
153
|
-
def run_item(item)
|
154
|
-
@item_skipper = false
|
155
|
-
|
156
|
-
return item if item.status(self.namepath) == :DONE
|
157
|
-
|
158
|
-
pre_process(item)
|
159
|
-
|
160
|
-
unless @item_skipper
|
161
|
-
set_status item, :STARTED
|
162
|
-
self.processing_item = item
|
163
|
-
self.process item
|
164
|
-
item = self.processing_item
|
165
|
-
run_subitems(item) if parameter(:recursive)
|
166
|
-
set_status item, :DONE if item.check_status(:STARTED, self.namepath)
|
167
|
-
else
|
168
|
-
run_subitems(item) if parameter(:recursive)
|
169
|
-
end
|
170
|
-
|
171
|
-
post_process item
|
172
|
-
|
173
|
-
item
|
174
|
-
end
|
175
|
-
|
176
|
-
def pre_process(_)
|
177
|
-
true
|
178
|
-
# optional implementation
|
179
|
-
end
|
180
|
-
|
181
|
-
def post_process(_)
|
182
|
-
# optional implementation
|
183
|
-
end
|
184
|
-
|
185
|
-
def run_subitems(parent_item)
|
186
|
-
return unless check_processing_subitems
|
187
|
-
|
188
|
-
items = subitems(parent_item)
|
189
|
-
return unless items.size > 0
|
190
|
-
|
191
|
-
status = Hash.new(0)
|
192
|
-
parent_item.status_progress(self.namepath, 0, items.count)
|
193
|
-
items.each_with_index do |item, i|
|
194
|
-
debug 'Processing subitem (%d/%d): %s', parent_item, i+1, items.size, item.to_s
|
195
|
-
item = run_item item
|
196
|
-
parent_item.status_progress(self.namepath, i+1)
|
197
|
-
item_status = item.status(self.namepath)
|
198
|
-
status[item_status] += 1
|
199
|
-
break if parameter(:abort_recursion_on_failure) && item_status != :DONE
|
200
|
-
end
|
201
|
-
|
202
|
-
debug '%d of %d subitems passed', parent_item, status[:DONE], items.size
|
203
|
-
substatus_check(status, parent_item, 'item')
|
204
|
-
end
|
205
|
-
|
206
|
-
def substatus_check(status, item, task_or_item)
|
207
|
-
item_status = :DONE
|
208
|
-
|
209
|
-
if (waiting = status[:ASYNC_WAIT]) > 0
|
210
|
-
info "waiting for %d sub#{task_or_item}(s) in async process", item, waiting
|
211
|
-
item_status = :ASYNC_WAIT
|
212
|
-
end
|
213
|
-
|
214
|
-
if (halted = status[:ASYNC_HALT]) > 0
|
215
|
-
warn "%d sub#{task_or_item}(s) halted in async process", item, halted
|
216
|
-
item_status = :ASYNC_HALT
|
217
|
-
end
|
218
|
-
|
219
|
-
if (failed = status[:FAILED]) > 0
|
220
|
-
error "%d sub#{task_or_item}(s) failed", item, failed
|
221
|
-
item_status = :FAILED
|
222
|
-
end
|
223
|
-
|
224
|
-
set_status(item, item_status)
|
225
|
-
end
|
226
|
-
|
227
|
-
def capture_cmd(cmd, *opts)
|
228
|
-
out = StringIO.new
|
229
|
-
err = StringIO.new
|
230
|
-
$stdout = out
|
231
|
-
$stderr = err
|
232
|
-
status = system cmd, *opts
|
233
|
-
return [status, out.string, err.string]
|
234
|
-
ensure
|
235
|
-
$stdout = STDOUT
|
236
|
-
$stderr = STDERR
|
237
|
-
end
|
238
|
-
|
239
|
-
def action=(action)
|
240
|
-
self.get_run.action = action
|
241
|
-
end
|
242
|
-
|
243
|
-
def action
|
244
|
-
self.get_run.action
|
245
|
-
end
|
246
|
-
|
247
|
-
def get_run(item = nil)
|
248
|
-
get_root_item(item).get_run
|
249
|
-
end
|
250
|
-
|
251
|
-
def get_root_item(item = nil)
|
252
|
-
(item || self.workitem).get_root
|
253
|
-
end
|
254
|
-
|
255
|
-
def get_work_dir(item = nil)
|
256
|
-
get_root_item(item).work_dir
|
257
|
-
end
|
258
|
-
|
259
|
-
def stop_processing_subitems
|
260
|
-
@subitems_stopper = true if parameter(:recursive)
|
261
|
-
end
|
262
|
-
|
263
|
-
def check_processing_subitems
|
264
|
-
if @subitems_stopper
|
265
|
-
@subitems_stopper = false
|
266
|
-
return false
|
267
|
-
end
|
268
|
-
true
|
269
|
-
end
|
270
|
-
|
271
|
-
def skip_processing_item
|
272
|
-
@item_skipper = true
|
273
|
-
end
|
274
|
-
|
275
|
-
def set_status(item, state)
|
276
|
-
item.set_status self.namepath, state
|
277
|
-
state
|
278
|
-
end
|
279
|
-
|
280
|
-
def check_item_type(klass, item = nil)
|
281
|
-
item ||= self.workitem
|
282
|
-
unless item.is_a? klass.to_s.constantize
|
283
|
-
raise WorkflowError, "Workitem is of wrong type : #{item.class} - expected #{klass.to_s}"
|
284
|
-
end
|
285
|
-
end
|
286
|
-
|
287
|
-
def item_type?(klass, item = nil)
|
288
|
-
item ||= self.workitem
|
289
|
-
item.is_a? klass.to_s.constantize
|
290
|
-
end
|
291
|
-
|
292
|
-
private
|
293
|
-
|
294
|
-
def subtasks
|
295
|
-
self.tasks
|
296
|
-
end
|
297
|
-
|
298
|
-
def subitems(item = nil)
|
299
|
-
(item || self.workitem).get_item_list
|
300
|
-
end
|
301
|
-
|
302
|
-
def default_values
|
303
|
-
self.class.default_values
|
304
|
-
end
|
305
|
-
|
306
|
-
def self.default_values
|
307
|
-
parameter_defs.inject({}) do |hash, parameter|
|
308
|
-
hash[parameter.first] = parameter.last[:default]
|
309
|
-
hash
|
310
|
-
end
|
311
|
-
end
|
312
|
-
|
313
|
-
end
|
314
|
-
|
315
|
-
end
|
316
|
-
end
|
1
|
+
require 'backports/rails/hash'
|
2
|
+
require 'backports/rails/string'
|
3
|
+
|
4
|
+
require 'libis/tools/parameter'
|
5
|
+
require 'libis/tools/extend/hash'
|
6
|
+
require 'libis/tools/logger'
|
7
|
+
|
8
|
+
require 'libis/workflow'
|
9
|
+
|
10
|
+
module Libis
|
11
|
+
module Workflow
|
12
|
+
|
13
|
+
# noinspection RubyTooManyMethodsInspection
|
14
|
+
class Task
|
15
|
+
include ::Libis::Tools::Logger
|
16
|
+
include ::Libis::Tools::ParameterContainer
|
17
|
+
|
18
|
+
attr_accessor :parent, :name, :workitem, :processing_item
|
19
|
+
|
20
|
+
parameter recursive: false, description: 'Run the task on all subitems recursively.'
|
21
|
+
parameter abort_recursion_on_failure: false, description: 'Stop processing items recursively if one item fails.'
|
22
|
+
parameter retry_count: 0, description: 'Number of times to retry the task if waiting for another process.'
|
23
|
+
parameter retry_interval: 10, description: 'Number of seconds to wait between retries.'
|
24
|
+
|
25
|
+
def self.task_classes
|
26
|
+
ObjectSpace.each_object(::Class).select { |klass| klass < self }
|
27
|
+
end
|
28
|
+
|
29
|
+
def initialize(parent, cfg = {})
|
30
|
+
@subitems_stopper = false
|
31
|
+
@subtasks_stopper = false
|
32
|
+
self.parent = parent
|
33
|
+
configure cfg
|
34
|
+
end
|
35
|
+
|
36
|
+
def <<(task)
|
37
|
+
raise Libis::WorkflowError, "Processing task '#{self.namepath}' is not allowed to have subtasks."
|
38
|
+
end
|
39
|
+
|
40
|
+
# @param [Libis::Workflow::Base::WorkItem] item
|
41
|
+
def run(item)
|
42
|
+
check_item_type ::Libis::Workflow::Base::WorkItem, item
|
43
|
+
self.workitem = item
|
44
|
+
|
45
|
+
case self.action
|
46
|
+
when :retry
|
47
|
+
if item.check_status(:DONE, self.namepath)
|
48
|
+
debug 'Retry: skipping task %s because it has finished successfully.', item, self.namepath
|
49
|
+
return item
|
50
|
+
end
|
51
|
+
when :failed
|
52
|
+
return item
|
53
|
+
else
|
54
|
+
end
|
55
|
+
|
56
|
+
(parameter(:retry_count)+1).times do
|
57
|
+
|
58
|
+
item = run_item(item)
|
59
|
+
|
60
|
+
case item.status(self.namepath)
|
61
|
+
when :DONE
|
62
|
+
self.action = :run
|
63
|
+
return item
|
64
|
+
when :ASYNC_WAIT
|
65
|
+
self.action = :retry
|
66
|
+
when :ASYNC_HALT
|
67
|
+
break
|
68
|
+
when :FAILED
|
69
|
+
break
|
70
|
+
else
|
71
|
+
return item
|
72
|
+
end
|
73
|
+
|
74
|
+
self.action = :retry
|
75
|
+
|
76
|
+
sleep(parameter(:retry_interval))
|
77
|
+
|
78
|
+
end
|
79
|
+
|
80
|
+
item.get_run.action = :failed
|
81
|
+
|
82
|
+
return item
|
83
|
+
|
84
|
+
rescue WorkflowError => e
|
85
|
+
error e.message, item
|
86
|
+
set_status item, :FAILED
|
87
|
+
|
88
|
+
rescue WorkflowAbort => e
|
89
|
+
set_status item, :FAILED
|
90
|
+
raise e if parent
|
91
|
+
|
92
|
+
rescue ::Exception => e
|
93
|
+
set_status item, :FAILED
|
94
|
+
fatal "Exception occured: #{e.message}", item
|
95
|
+
debug e.backtrace.join("\n")
|
96
|
+
|
97
|
+
ensure
|
98
|
+
item.save!
|
99
|
+
|
100
|
+
end
|
101
|
+
|
102
|
+
def names
|
103
|
+
(self.parent.names rescue Array.new).push(name).compact
|
104
|
+
end
|
105
|
+
|
106
|
+
def namepath;
|
107
|
+
self.names.join('/');
|
108
|
+
end
|
109
|
+
|
110
|
+
def apply_options(opts)
|
111
|
+
o = {}
|
112
|
+
o.merge!(opts[self.class.to_s] || {})
|
113
|
+
o.merge!(opts[self.name] || opts[self.names.join('/')] || {})
|
114
|
+
|
115
|
+
if o and o.is_a? Hash
|
116
|
+
default_values.each do |name, _|
|
117
|
+
next unless o.key?(name.to_s)
|
118
|
+
next if o[name.to_s].nil?
|
119
|
+
paramdef = get_parameter_definition name.to_sym
|
120
|
+
value = paramdef.parse(o[name.to_s])
|
121
|
+
self.parameter(name.to_sym, value)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def message(severity, msg, *args)
|
127
|
+
taskname = self.namepath rescue nil
|
128
|
+
self.set_application(taskname)
|
129
|
+
item = self.workitem rescue nil
|
130
|
+
item = args.shift if args.size > 0 and args[0].is_a?(::Libis::Workflow::Base::WorkItem)
|
131
|
+
subject = item.namepath rescue nil
|
132
|
+
subject ||= item.name rescue nil
|
133
|
+
subject ||= item.to_s rescue nil
|
134
|
+
self.set_subject(subject)
|
135
|
+
super(severity, msg, *args)
|
136
|
+
end
|
137
|
+
|
138
|
+
def logger
|
139
|
+
(self.parent || self.get_run).logger
|
140
|
+
end
|
141
|
+
|
142
|
+
protected
|
143
|
+
|
144
|
+
def configure(cfg)
|
145
|
+
self.name = cfg['name'] || (cfg['class'] || self.class).to_s.split('::').last
|
146
|
+
(cfg['options'] || {}).merge(
|
147
|
+
cfg.reject { |k, _| %w(options name class).include? k }
|
148
|
+
).symbolize_keys.each do |k, v|
|
149
|
+
self.parameter(k, v)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
def run_item(item)
|
154
|
+
@item_skipper = false
|
155
|
+
|
156
|
+
return item if item.status(self.namepath) == :DONE
|
157
|
+
|
158
|
+
pre_process(item)
|
159
|
+
|
160
|
+
unless @item_skipper
|
161
|
+
set_status item, :STARTED
|
162
|
+
self.processing_item = item
|
163
|
+
self.process item
|
164
|
+
item = self.processing_item
|
165
|
+
run_subitems(item) if parameter(:recursive)
|
166
|
+
set_status item, :DONE if item.check_status(:STARTED, self.namepath)
|
167
|
+
else
|
168
|
+
run_subitems(item) if parameter(:recursive)
|
169
|
+
end
|
170
|
+
|
171
|
+
post_process item
|
172
|
+
|
173
|
+
item
|
174
|
+
end
|
175
|
+
|
176
|
+
def pre_process(_)
|
177
|
+
true
|
178
|
+
# optional implementation
|
179
|
+
end
|
180
|
+
|
181
|
+
def post_process(_)
|
182
|
+
# optional implementation
|
183
|
+
end
|
184
|
+
|
185
|
+
def run_subitems(parent_item)
|
186
|
+
return unless check_processing_subitems
|
187
|
+
|
188
|
+
items = subitems(parent_item)
|
189
|
+
return unless items.size > 0
|
190
|
+
|
191
|
+
status = Hash.new(0)
|
192
|
+
parent_item.status_progress(self.namepath, 0, items.count)
|
193
|
+
items.each_with_index do |item, i|
|
194
|
+
debug 'Processing subitem (%d/%d): %s', parent_item, i+1, items.size, item.to_s
|
195
|
+
item = run_item item
|
196
|
+
parent_item.status_progress(self.namepath, i+1)
|
197
|
+
item_status = item.status(self.namepath)
|
198
|
+
status[item_status] += 1
|
199
|
+
break if parameter(:abort_recursion_on_failure) && item_status != :DONE
|
200
|
+
end
|
201
|
+
|
202
|
+
debug '%d of %d subitems passed', parent_item, status[:DONE], items.size
|
203
|
+
substatus_check(status, parent_item, 'item')
|
204
|
+
end
|
205
|
+
|
206
|
+
def substatus_check(status, item, task_or_item)
|
207
|
+
item_status = :DONE
|
208
|
+
|
209
|
+
if (waiting = status[:ASYNC_WAIT]) > 0
|
210
|
+
info "waiting for %d sub#{task_or_item}(s) in async process", item, waiting
|
211
|
+
item_status = :ASYNC_WAIT
|
212
|
+
end
|
213
|
+
|
214
|
+
if (halted = status[:ASYNC_HALT]) > 0
|
215
|
+
warn "%d sub#{task_or_item}(s) halted in async process", item, halted
|
216
|
+
item_status = :ASYNC_HALT
|
217
|
+
end
|
218
|
+
|
219
|
+
if (failed = status[:FAILED]) > 0
|
220
|
+
error "%d sub#{task_or_item}(s) failed", item, failed
|
221
|
+
item_status = :FAILED
|
222
|
+
end
|
223
|
+
|
224
|
+
set_status(item, item_status)
|
225
|
+
end
|
226
|
+
|
227
|
+
def capture_cmd(cmd, *opts)
|
228
|
+
out = StringIO.new
|
229
|
+
err = StringIO.new
|
230
|
+
$stdout = out
|
231
|
+
$stderr = err
|
232
|
+
status = system cmd, *opts
|
233
|
+
return [status, out.string, err.string]
|
234
|
+
ensure
|
235
|
+
$stdout = STDOUT
|
236
|
+
$stderr = STDERR
|
237
|
+
end
|
238
|
+
|
239
|
+
def action=(action)
|
240
|
+
self.get_run.action = action
|
241
|
+
end
|
242
|
+
|
243
|
+
def action
|
244
|
+
self.get_run.action
|
245
|
+
end
|
246
|
+
|
247
|
+
def get_run(item = nil)
|
248
|
+
get_root_item(item).get_run
|
249
|
+
end
|
250
|
+
|
251
|
+
def get_root_item(item = nil)
|
252
|
+
(item || self.workitem).get_root
|
253
|
+
end
|
254
|
+
|
255
|
+
def get_work_dir(item = nil)
|
256
|
+
get_root_item(item).work_dir
|
257
|
+
end
|
258
|
+
|
259
|
+
def stop_processing_subitems
|
260
|
+
@subitems_stopper = true if parameter(:recursive)
|
261
|
+
end
|
262
|
+
|
263
|
+
def check_processing_subitems
|
264
|
+
if @subitems_stopper
|
265
|
+
@subitems_stopper = false
|
266
|
+
return false
|
267
|
+
end
|
268
|
+
true
|
269
|
+
end
|
270
|
+
|
271
|
+
def skip_processing_item
|
272
|
+
@item_skipper = true
|
273
|
+
end
|
274
|
+
|
275
|
+
def set_status(item, state)
|
276
|
+
item.set_status self.namepath, state
|
277
|
+
state
|
278
|
+
end
|
279
|
+
|
280
|
+
def check_item_type(klass, item = nil)
|
281
|
+
item ||= self.workitem
|
282
|
+
unless item.is_a? klass.to_s.constantize
|
283
|
+
raise WorkflowError, "Workitem is of wrong type : #{item.class} - expected #{klass.to_s}"
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
def item_type?(klass, item = nil)
|
288
|
+
item ||= self.workitem
|
289
|
+
item.is_a? klass.to_s.constantize
|
290
|
+
end
|
291
|
+
|
292
|
+
private
|
293
|
+
|
294
|
+
def subtasks
|
295
|
+
self.tasks
|
296
|
+
end
|
297
|
+
|
298
|
+
def subitems(item = nil)
|
299
|
+
(item || self.workitem).get_item_list
|
300
|
+
end
|
301
|
+
|
302
|
+
def default_values
|
303
|
+
self.class.default_values
|
304
|
+
end
|
305
|
+
|
306
|
+
def self.default_values
|
307
|
+
parameter_defs.inject({}) do |hash, parameter|
|
308
|
+
hash[parameter.first] = parameter.last[:default]
|
309
|
+
hash
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
313
|
+
end
|
314
|
+
|
315
|
+
end
|
316
|
+
end
|