dynflow 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +6 -0
- data/.travis.yml +9 -0
- data/Gemfile +0 -10
- data/MIT-LICENSE +1 -1
- data/README.md +99 -37
- data/Rakefile +2 -6
- data/doc/images/logo.png +0 -0
- data/dynflow.gemspec +10 -1
- data/examples/generate_work_for_daemon.rb +24 -0
- data/examples/orchestrate.rb +121 -0
- data/examples/run_daemon.rb +17 -0
- data/examples/web_console.rb +29 -0
- data/lib/dynflow.rb +27 -6
- data/lib/dynflow/action.rb +185 -77
- data/lib/dynflow/action/cancellable_polling.rb +18 -0
- data/lib/dynflow/action/finalize_phase.rb +18 -0
- data/lib/dynflow/action/flow_phase.rb +44 -0
- data/lib/dynflow/action/format.rb +46 -0
- data/lib/dynflow/action/missing.rb +26 -0
- data/lib/dynflow/action/plan_phase.rb +85 -0
- data/lib/dynflow/action/polling.rb +49 -0
- data/lib/dynflow/action/presenter.rb +51 -0
- data/lib/dynflow/action/progress.rb +62 -0
- data/lib/dynflow/action/run_phase.rb +43 -0
- data/lib/dynflow/action/suspended.rb +21 -0
- data/lib/dynflow/clock.rb +133 -0
- data/lib/dynflow/daemon.rb +29 -0
- data/lib/dynflow/execution_plan.rb +285 -33
- data/lib/dynflow/execution_plan/dependency_graph.rb +29 -0
- data/lib/dynflow/execution_plan/output_reference.rb +52 -0
- data/lib/dynflow/execution_plan/steps.rb +12 -0
- data/lib/dynflow/execution_plan/steps/abstract.rb +121 -0
- data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb +52 -0
- data/lib/dynflow/execution_plan/steps/error.rb +33 -0
- data/lib/dynflow/execution_plan/steps/finalize_step.rb +23 -0
- data/lib/dynflow/execution_plan/steps/plan_step.rb +81 -0
- data/lib/dynflow/execution_plan/steps/run_step.rb +21 -0
- data/lib/dynflow/executors.rb +9 -0
- data/lib/dynflow/executors/abstract.rb +32 -0
- data/lib/dynflow/executors/parallel.rb +88 -0
- data/lib/dynflow/executors/parallel/core.rb +119 -0
- data/lib/dynflow/executors/parallel/execution_plan_manager.rb +120 -0
- data/lib/dynflow/executors/parallel/flow_manager.rb +48 -0
- data/lib/dynflow/executors/parallel/pool.rb +102 -0
- data/lib/dynflow/executors/parallel/running_steps_manager.rb +63 -0
- data/lib/dynflow/executors/parallel/sequence_cursor.rb +97 -0
- data/lib/dynflow/executors/parallel/sequential_manager.rb +81 -0
- data/lib/dynflow/executors/parallel/work_queue.rb +44 -0
- data/lib/dynflow/executors/parallel/worker.rb +30 -0
- data/lib/dynflow/executors/remote_via_socket.rb +38 -0
- data/lib/dynflow/executors/remote_via_socket/core.rb +150 -0
- data/lib/dynflow/flows.rb +13 -0
- data/lib/dynflow/flows/abstract.rb +36 -0
- data/lib/dynflow/flows/abstract_composed.rb +104 -0
- data/lib/dynflow/flows/atom.rb +36 -0
- data/lib/dynflow/flows/concurrence.rb +28 -0
- data/lib/dynflow/flows/sequence.rb +13 -0
- data/lib/dynflow/future.rb +173 -0
- data/lib/dynflow/listeners.rb +7 -0
- data/lib/dynflow/listeners/abstract.rb +13 -0
- data/lib/dynflow/listeners/serialization.rb +41 -0
- data/lib/dynflow/listeners/socket.rb +88 -0
- data/lib/dynflow/logger_adapters.rb +8 -0
- data/lib/dynflow/logger_adapters/abstract.rb +30 -0
- data/lib/dynflow/logger_adapters/delegator.rb +13 -0
- data/lib/dynflow/logger_adapters/formatters.rb +8 -0
- data/lib/dynflow/logger_adapters/formatters/abstract.rb +33 -0
- data/lib/dynflow/logger_adapters/formatters/exception.rb +15 -0
- data/lib/dynflow/logger_adapters/simple.rb +59 -0
- data/lib/dynflow/micro_actor.rb +102 -0
- data/lib/dynflow/persistence.rb +53 -0
- data/lib/dynflow/persistence_adapters.rb +6 -0
- data/lib/dynflow/persistence_adapters/abstract.rb +56 -0
- data/lib/dynflow/persistence_adapters/sequel.rb +160 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/001_initial.rb +52 -0
- data/lib/dynflow/serializable.rb +66 -0
- data/lib/dynflow/simple_world.rb +18 -0
- data/lib/dynflow/stateful.rb +40 -0
- data/lib/dynflow/testing.rb +32 -0
- data/lib/dynflow/testing/assertions.rb +64 -0
- data/lib/dynflow/testing/dummy_execution_plan.rb +40 -0
- data/lib/dynflow/testing/dummy_executor.rb +29 -0
- data/lib/dynflow/testing/dummy_planned_action.rb +18 -0
- data/lib/dynflow/testing/dummy_step.rb +19 -0
- data/lib/dynflow/testing/dummy_world.rb +33 -0
- data/lib/dynflow/testing/factories.rb +83 -0
- data/lib/dynflow/testing/managed_clock.rb +23 -0
- data/lib/dynflow/testing/mimic.rb +38 -0
- data/lib/dynflow/transaction_adapters.rb +9 -0
- data/lib/dynflow/transaction_adapters/abstract.rb +26 -0
- data/lib/dynflow/transaction_adapters/active_record.rb +27 -0
- data/lib/dynflow/transaction_adapters/none.rb +12 -0
- data/lib/dynflow/version.rb +1 -1
- data/lib/dynflow/web_console.rb +277 -0
- data/lib/dynflow/world.rb +168 -0
- data/test/action_test.rb +89 -11
- data/test/clock_test.rb +59 -0
- data/test/code_workflow_example.rb +382 -0
- data/test/execution_plan_test.rb +195 -64
- data/test/executor_test.rb +692 -0
- data/test/persistance_adapters_test.rb +173 -0
- data/test/test_helper.rb +316 -1
- data/test/testing_test.rb +148 -0
- data/test/web_console_test.rb +38 -0
- data/web/assets/javascripts/application.js +25 -0
- data/web/assets/stylesheets/application.css +101 -0
- data/web/assets/vendor/bootstrap/css/bootstrap-responsive.css +1109 -0
- data/web/assets/vendor/bootstrap/css/bootstrap-responsive.min.css +9 -0
- data/web/assets/vendor/bootstrap/css/bootstrap.css +6167 -0
- data/web/assets/vendor/bootstrap/css/bootstrap.min.css +9 -0
- data/web/assets/vendor/bootstrap/img/glyphicons-halflings-white.png +0 -0
- data/web/assets/vendor/bootstrap/img/glyphicons-halflings.png +0 -0
- data/web/assets/vendor/bootstrap/js/bootstrap.js +2280 -0
- data/web/assets/vendor/bootstrap/js/bootstrap.min.js +6 -0
- data/web/assets/vendor/google-code-prettify/lang-basic.js +3 -0
- data/web/assets/vendor/google-code-prettify/prettify.css +1 -0
- data/web/assets/vendor/google-code-prettify/prettify.js +30 -0
- data/web/assets/vendor/google-code-prettify/run_prettify.js +34 -0
- data/web/assets/vendor/jquery/jquery.js +9807 -0
- data/web/views/flow.erb +19 -0
- data/web/views/flow_step.erb +31 -0
- data/web/views/index.erb +39 -0
- data/web/views/layout.erb +20 -0
- data/web/views/plan_step.erb +11 -0
- data/web/views/show.erb +54 -0
- metadata +250 -11
- data/examples/events.rb +0 -71
- data/examples/workflow.rb +0 -140
- data/lib/dynflow/bus.rb +0 -168
- data/lib/dynflow/dispatcher.rb +0 -36
- data/lib/dynflow/logger.rb +0 -34
- data/lib/dynflow/step.rb +0 -234
- data/test/bus_test.rb +0 -150
data/lib/dynflow/step.rb
DELETED
@@ -1,234 +0,0 @@
|
|
1
|
-
module Dynflow
|
2
|
-
class Step
|
3
|
-
|
4
|
-
class Reference
|
5
|
-
|
6
|
-
attr_reader :step, :field
|
7
|
-
|
8
|
-
def initialize(step, field)
|
9
|
-
unless %w[input output].include? field
|
10
|
-
raise "Unexpected reference field: #{field}. Only input and output allowed"
|
11
|
-
end
|
12
|
-
@step = step
|
13
|
-
@field = field
|
14
|
-
end
|
15
|
-
|
16
|
-
def encode
|
17
|
-
unless @step.persistence
|
18
|
-
raise "Reference can't be serialized without persistence available"
|
19
|
-
end
|
20
|
-
|
21
|
-
{
|
22
|
-
'dynflow_step_persistence_id' => @step.persistence.persistence_id,
|
23
|
-
'field' => @field
|
24
|
-
}
|
25
|
-
end
|
26
|
-
|
27
|
-
def self.decode(data)
|
28
|
-
return nil unless data.is_a? Hash
|
29
|
-
return nil unless data.has_key?('dynflow_step_persistence_id')
|
30
|
-
persistence_id = data['dynflow_step_persistence_id']
|
31
|
-
self.new(Dynflow::Bus.persisted_step(persistence_id), data['field'])
|
32
|
-
end
|
33
|
-
|
34
|
-
def dereference
|
35
|
-
@step.send(@field)
|
36
|
-
end
|
37
|
-
|
38
|
-
def inspect
|
39
|
-
ret = "References "
|
40
|
-
ret << @step.class.name.split('::').last
|
41
|
-
ret << "/"
|
42
|
-
ret << @step.action_class.name
|
43
|
-
ret << "(#{@step.persistence.persistence_id})" if @step.persistence
|
44
|
-
ret << "/"
|
45
|
-
ret << @field
|
46
|
-
return ret
|
47
|
-
end
|
48
|
-
|
49
|
-
end
|
50
|
-
|
51
|
-
extend Forwardable
|
52
|
-
|
53
|
-
def_delegators :@data, '[]', '[]='
|
54
|
-
|
55
|
-
attr_accessor :status
|
56
|
-
attr_reader :data, :action_class
|
57
|
-
|
58
|
-
# persistent representation of the step
|
59
|
-
attr_accessor :persistence
|
60
|
-
|
61
|
-
def input
|
62
|
-
@data['input']
|
63
|
-
end
|
64
|
-
|
65
|
-
def input=(input)
|
66
|
-
@data['input'] = input
|
67
|
-
end
|
68
|
-
|
69
|
-
def output
|
70
|
-
@data['output']
|
71
|
-
end
|
72
|
-
|
73
|
-
def output=(output)
|
74
|
-
@data['output'] = output
|
75
|
-
end
|
76
|
-
|
77
|
-
def error
|
78
|
-
@data['error']
|
79
|
-
end
|
80
|
-
|
81
|
-
def error=(error)
|
82
|
-
@data['error'] = error
|
83
|
-
end
|
84
|
-
|
85
|
-
# get a fresh instance of action class for execution
|
86
|
-
def action
|
87
|
-
self.action_class.new(input, output)
|
88
|
-
end
|
89
|
-
|
90
|
-
def catch_errors
|
91
|
-
yield
|
92
|
-
self.status = 'success'
|
93
|
-
return true
|
94
|
-
rescue Exception => e
|
95
|
-
self.error = {
|
96
|
-
"exception" => e.class.name,
|
97
|
-
"message" => e.message,
|
98
|
-
"backtrace" => e.backtrace
|
99
|
-
}
|
100
|
-
self.status = 'error'
|
101
|
-
return false
|
102
|
-
end
|
103
|
-
|
104
|
-
def ==(other)
|
105
|
-
self.encode == other.encode
|
106
|
-
end
|
107
|
-
|
108
|
-
def self.decode(data)
|
109
|
-
ret = data['step_class'].constantize.allocate
|
110
|
-
ret.instance_variable_set("@action_class", data['action_class'].constantize)
|
111
|
-
ret.instance_variable_set("@status", data['status'])
|
112
|
-
ret.instance_variable_set("@data", decode_data(data['data']))
|
113
|
-
return ret
|
114
|
-
end
|
115
|
-
|
116
|
-
def encode
|
117
|
-
{
|
118
|
-
'step_class' => self.class.name,
|
119
|
-
'action_class' => action_class.name,
|
120
|
-
'status' => status,
|
121
|
-
'data' => encoded_data
|
122
|
-
}
|
123
|
-
end
|
124
|
-
|
125
|
-
def self.decode_data(data)
|
126
|
-
walk(data) do |item|
|
127
|
-
Reference.decode(item)
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
# we need this to encode the reference correctly
|
132
|
-
def encoded_data
|
133
|
-
self.class.walk(data) do |item|
|
134
|
-
if item.is_a? Reference
|
135
|
-
item.encode
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
|
-
def replace_references!
|
141
|
-
@data = self.class.walk(data) do |item|
|
142
|
-
if item.is_a? Reference
|
143
|
-
if item.step.status == 'skipped' || item.step.status == 'error'
|
144
|
-
self.status = 'skipped'
|
145
|
-
item
|
146
|
-
else
|
147
|
-
item.dereference
|
148
|
-
end
|
149
|
-
end
|
150
|
-
end
|
151
|
-
end
|
152
|
-
|
153
|
-
# walks hash depth-first, yielding on every value
|
154
|
-
# if yield return non-false value, use that instead of original
|
155
|
-
# value in a resulting hash
|
156
|
-
def self.walk(data, &block)
|
157
|
-
if converted = (yield data)
|
158
|
-
return converted
|
159
|
-
end
|
160
|
-
case data
|
161
|
-
when Array
|
162
|
-
data.map { |d| walk(d, &block) }
|
163
|
-
when Hash
|
164
|
-
data.reduce({}) { |h, (k, v)| h.update(k => walk(v, &block)) }
|
165
|
-
else
|
166
|
-
data
|
167
|
-
end
|
168
|
-
end
|
169
|
-
|
170
|
-
def persist
|
171
|
-
if @persistence
|
172
|
-
@persistence.persist(self)
|
173
|
-
end
|
174
|
-
end
|
175
|
-
|
176
|
-
def persist_before_run
|
177
|
-
if @persistence
|
178
|
-
@persistence.before_run(self)
|
179
|
-
end
|
180
|
-
end
|
181
|
-
|
182
|
-
def persist_after_run
|
183
|
-
if @persistence
|
184
|
-
@persistence.after_run(self)
|
185
|
-
end
|
186
|
-
end
|
187
|
-
|
188
|
-
class Plan < Step
|
189
|
-
|
190
|
-
def initialize(action)
|
191
|
-
# we want to have the steps separated:
|
192
|
-
# not using the original action object
|
193
|
-
@action_class = action.class
|
194
|
-
self.status = 'finished' # default status
|
195
|
-
@data = {}.with_indifferent_access
|
196
|
-
end
|
197
|
-
|
198
|
-
end
|
199
|
-
|
200
|
-
class Run < Step
|
201
|
-
|
202
|
-
def initialize(action)
|
203
|
-
# we want to have the steps separated:
|
204
|
-
# not using the original action object
|
205
|
-
@action_class = action.class
|
206
|
-
self.status = 'pending' # default status
|
207
|
-
@data = {
|
208
|
-
'input' => action.input,
|
209
|
-
'output' => action.output
|
210
|
-
}.with_indifferent_access
|
211
|
-
end
|
212
|
-
|
213
|
-
end
|
214
|
-
|
215
|
-
class Finalize < Step
|
216
|
-
|
217
|
-
def initialize(run_step)
|
218
|
-
# we want to have the steps separated:
|
219
|
-
# not using the original action object
|
220
|
-
@action_class = run_step.action_class
|
221
|
-
self.status = 'pending' # default status
|
222
|
-
if run_step.action.respond_to? :run
|
223
|
-
@data = {
|
224
|
-
'input' => Reference.new(run_step, 'input'),
|
225
|
-
'output' => Reference.new(run_step, 'output'),
|
226
|
-
}
|
227
|
-
else
|
228
|
-
@data = run_step.data
|
229
|
-
end
|
230
|
-
end
|
231
|
-
|
232
|
-
end
|
233
|
-
end
|
234
|
-
end
|
data/test/bus_test.rb
DELETED
@@ -1,150 +0,0 @@
|
|
1
|
-
require 'test_helper'
|
2
|
-
require 'set'
|
3
|
-
|
4
|
-
module Dynflow
|
5
|
-
module BusTest
|
6
|
-
describe "bus" do
|
7
|
-
|
8
|
-
class Promotion < Action
|
9
|
-
|
10
|
-
def plan(repo_names, package_names)
|
11
|
-
repo_names.each do |repo_name|
|
12
|
-
plan_action(CloneRepo, {'name' => repo_name})
|
13
|
-
end
|
14
|
-
|
15
|
-
package_names.each do |package_name|
|
16
|
-
plan_action(ClonePackage, {'name' => package_name})
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
end
|
21
|
-
|
22
|
-
class CloneRepo < Action
|
23
|
-
|
24
|
-
input_format do
|
25
|
-
param :name, String
|
26
|
-
end
|
27
|
-
|
28
|
-
output_format do
|
29
|
-
param :id, String
|
30
|
-
end
|
31
|
-
|
32
|
-
def plan(input)
|
33
|
-
raise 'Simulate error in plan phase' if input['name'] == 'fail_in_plan'
|
34
|
-
plan_self(input)
|
35
|
-
end
|
36
|
-
|
37
|
-
def run
|
38
|
-
raise 'Simulate error in execution phase' if input['name'] == 'fail_in_run'
|
39
|
-
output['id'] = input['name']
|
40
|
-
end
|
41
|
-
|
42
|
-
def finalize(outputs)
|
43
|
-
raise 'Simulate error in finalize phase' if input['name'] == 'fail_in_finalize'
|
44
|
-
end
|
45
|
-
|
46
|
-
end
|
47
|
-
|
48
|
-
it 'returns the execution plan obejct when triggering an action' do
|
49
|
-
Promotion.trigger(['sucess'], []).must_be_instance_of Dynflow::ExecutionPlan
|
50
|
-
end
|
51
|
-
|
52
|
-
describe 'handling errros in plan phase' do
|
53
|
-
|
54
|
-
let(:failed_plan) { Promotion.trigger(['fail_in_plan'], []) }
|
55
|
-
let(:failed_step) { failed_plan.plan_steps.last }
|
56
|
-
|
57
|
-
it 'marks the process as error' do
|
58
|
-
failed_plan.status.must_equal 'error'
|
59
|
-
end
|
60
|
-
|
61
|
-
it 'saves errors of actions' do
|
62
|
-
failed_step.status.must_equal "error"
|
63
|
-
expected_error = {
|
64
|
-
'exception' => 'RuntimeError',
|
65
|
-
'message' => 'Simulate error in plan phase'
|
66
|
-
}
|
67
|
-
failed_step.error['exception'].must_equal expected_error['exception']
|
68
|
-
failed_step.error['message'].must_equal expected_error['message']
|
69
|
-
failed_step.error['backtrace'].must_be_instance_of Array
|
70
|
-
end
|
71
|
-
|
72
|
-
end
|
73
|
-
|
74
|
-
describe 'handling errros in execution phase' do
|
75
|
-
|
76
|
-
let(:failed_plan) { Promotion.trigger(['fail_in_run'], []) }
|
77
|
-
let(:failed_step) { failed_plan.run_steps.first }
|
78
|
-
|
79
|
-
it 'pauses the process' do
|
80
|
-
failed_plan.status.must_equal 'paused'
|
81
|
-
end
|
82
|
-
|
83
|
-
it 'saves errors of actions' do
|
84
|
-
failed_step.status.must_equal "error"
|
85
|
-
expected_error = {
|
86
|
-
'exception' => 'RuntimeError',
|
87
|
-
'message' => 'Simulate error in execution phase'
|
88
|
-
}
|
89
|
-
failed_step.error['exception'].must_equal expected_error['exception']
|
90
|
-
failed_step.error['message'].must_equal expected_error['message']
|
91
|
-
failed_step.error['backtrace'].must_be_instance_of Array
|
92
|
-
end
|
93
|
-
|
94
|
-
it 'allows skipping the step' do
|
95
|
-
Dynflow::Bus.skip(failed_step)
|
96
|
-
Dynflow::Bus.resume(failed_plan)
|
97
|
-
|
98
|
-
failed_plan.status.must_equal 'finished'
|
99
|
-
failed_step.status.must_equal 'skipped'
|
100
|
-
end
|
101
|
-
|
102
|
-
it 'allows rerunning the step' do
|
103
|
-
failed_step.input['name'] = 'succeed'
|
104
|
-
Dynflow::Bus.resume(failed_plan)
|
105
|
-
|
106
|
-
failed_plan.status.must_equal 'finished'
|
107
|
-
failed_step.output.must_equal('id' => 'succeed')
|
108
|
-
end
|
109
|
-
|
110
|
-
end
|
111
|
-
|
112
|
-
describe 'handling errors in finalizatoin phase' do
|
113
|
-
|
114
|
-
let(:failed_plan) { Promotion.trigger(['fail_in_finalize'], []) }
|
115
|
-
let(:failed_step) { failed_plan.finalize_steps.first }
|
116
|
-
|
117
|
-
it 'pauses the process' do
|
118
|
-
failed_plan.status.must_equal 'paused'
|
119
|
-
end
|
120
|
-
|
121
|
-
it 'saves errors of actions' do
|
122
|
-
expected_error = {
|
123
|
-
'exception' => 'RuntimeError',
|
124
|
-
'message' => 'Simulate error in finalize phase'
|
125
|
-
}
|
126
|
-
failed_step.error['exception'].must_equal expected_error['exception']
|
127
|
-
failed_step.error['message'].must_equal expected_error['message']
|
128
|
-
failed_step.error['backtrace'].must_be_instance_of Array
|
129
|
-
end
|
130
|
-
|
131
|
-
it 'allows finishing a finalize phase' do
|
132
|
-
failed_step.input['name'] = 'succeed'
|
133
|
-
Dynflow::Bus.resume(failed_plan)
|
134
|
-
|
135
|
-
failed_plan.status.must_equal 'finished'
|
136
|
-
end
|
137
|
-
|
138
|
-
it 'allows skipping the step' do
|
139
|
-
Dynflow::Bus.skip(failed_step)
|
140
|
-
Dynflow::Bus.resume(failed_plan)
|
141
|
-
|
142
|
-
failed_plan.status.must_equal 'finished'
|
143
|
-
failed_step.status.must_equal 'skipped'
|
144
|
-
end
|
145
|
-
|
146
|
-
end
|
147
|
-
|
148
|
-
end
|
149
|
-
end
|
150
|
-
end
|