dynflow 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. data/.gitignore +6 -0
  2. data/.travis.yml +9 -0
  3. data/Gemfile +0 -10
  4. data/MIT-LICENSE +1 -1
  5. data/README.md +99 -37
  6. data/Rakefile +2 -6
  7. data/doc/images/logo.png +0 -0
  8. data/dynflow.gemspec +10 -1
  9. data/examples/generate_work_for_daemon.rb +24 -0
  10. data/examples/orchestrate.rb +121 -0
  11. data/examples/run_daemon.rb +17 -0
  12. data/examples/web_console.rb +29 -0
  13. data/lib/dynflow.rb +27 -6
  14. data/lib/dynflow/action.rb +185 -77
  15. data/lib/dynflow/action/cancellable_polling.rb +18 -0
  16. data/lib/dynflow/action/finalize_phase.rb +18 -0
  17. data/lib/dynflow/action/flow_phase.rb +44 -0
  18. data/lib/dynflow/action/format.rb +46 -0
  19. data/lib/dynflow/action/missing.rb +26 -0
  20. data/lib/dynflow/action/plan_phase.rb +85 -0
  21. data/lib/dynflow/action/polling.rb +49 -0
  22. data/lib/dynflow/action/presenter.rb +51 -0
  23. data/lib/dynflow/action/progress.rb +62 -0
  24. data/lib/dynflow/action/run_phase.rb +43 -0
  25. data/lib/dynflow/action/suspended.rb +21 -0
  26. data/lib/dynflow/clock.rb +133 -0
  27. data/lib/dynflow/daemon.rb +29 -0
  28. data/lib/dynflow/execution_plan.rb +285 -33
  29. data/lib/dynflow/execution_plan/dependency_graph.rb +29 -0
  30. data/lib/dynflow/execution_plan/output_reference.rb +52 -0
  31. data/lib/dynflow/execution_plan/steps.rb +12 -0
  32. data/lib/dynflow/execution_plan/steps/abstract.rb +121 -0
  33. data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb +52 -0
  34. data/lib/dynflow/execution_plan/steps/error.rb +33 -0
  35. data/lib/dynflow/execution_plan/steps/finalize_step.rb +23 -0
  36. data/lib/dynflow/execution_plan/steps/plan_step.rb +81 -0
  37. data/lib/dynflow/execution_plan/steps/run_step.rb +21 -0
  38. data/lib/dynflow/executors.rb +9 -0
  39. data/lib/dynflow/executors/abstract.rb +32 -0
  40. data/lib/dynflow/executors/parallel.rb +88 -0
  41. data/lib/dynflow/executors/parallel/core.rb +119 -0
  42. data/lib/dynflow/executors/parallel/execution_plan_manager.rb +120 -0
  43. data/lib/dynflow/executors/parallel/flow_manager.rb +48 -0
  44. data/lib/dynflow/executors/parallel/pool.rb +102 -0
  45. data/lib/dynflow/executors/parallel/running_steps_manager.rb +63 -0
  46. data/lib/dynflow/executors/parallel/sequence_cursor.rb +97 -0
  47. data/lib/dynflow/executors/parallel/sequential_manager.rb +81 -0
  48. data/lib/dynflow/executors/parallel/work_queue.rb +44 -0
  49. data/lib/dynflow/executors/parallel/worker.rb +30 -0
  50. data/lib/dynflow/executors/remote_via_socket.rb +38 -0
  51. data/lib/dynflow/executors/remote_via_socket/core.rb +150 -0
  52. data/lib/dynflow/flows.rb +13 -0
  53. data/lib/dynflow/flows/abstract.rb +36 -0
  54. data/lib/dynflow/flows/abstract_composed.rb +104 -0
  55. data/lib/dynflow/flows/atom.rb +36 -0
  56. data/lib/dynflow/flows/concurrence.rb +28 -0
  57. data/lib/dynflow/flows/sequence.rb +13 -0
  58. data/lib/dynflow/future.rb +173 -0
  59. data/lib/dynflow/listeners.rb +7 -0
  60. data/lib/dynflow/listeners/abstract.rb +13 -0
  61. data/lib/dynflow/listeners/serialization.rb +41 -0
  62. data/lib/dynflow/listeners/socket.rb +88 -0
  63. data/lib/dynflow/logger_adapters.rb +8 -0
  64. data/lib/dynflow/logger_adapters/abstract.rb +30 -0
  65. data/lib/dynflow/logger_adapters/delegator.rb +13 -0
  66. data/lib/dynflow/logger_adapters/formatters.rb +8 -0
  67. data/lib/dynflow/logger_adapters/formatters/abstract.rb +33 -0
  68. data/lib/dynflow/logger_adapters/formatters/exception.rb +15 -0
  69. data/lib/dynflow/logger_adapters/simple.rb +59 -0
  70. data/lib/dynflow/micro_actor.rb +102 -0
  71. data/lib/dynflow/persistence.rb +53 -0
  72. data/lib/dynflow/persistence_adapters.rb +6 -0
  73. data/lib/dynflow/persistence_adapters/abstract.rb +56 -0
  74. data/lib/dynflow/persistence_adapters/sequel.rb +160 -0
  75. data/lib/dynflow/persistence_adapters/sequel_migrations/001_initial.rb +52 -0
  76. data/lib/dynflow/serializable.rb +66 -0
  77. data/lib/dynflow/simple_world.rb +18 -0
  78. data/lib/dynflow/stateful.rb +40 -0
  79. data/lib/dynflow/testing.rb +32 -0
  80. data/lib/dynflow/testing/assertions.rb +64 -0
  81. data/lib/dynflow/testing/dummy_execution_plan.rb +40 -0
  82. data/lib/dynflow/testing/dummy_executor.rb +29 -0
  83. data/lib/dynflow/testing/dummy_planned_action.rb +18 -0
  84. data/lib/dynflow/testing/dummy_step.rb +19 -0
  85. data/lib/dynflow/testing/dummy_world.rb +33 -0
  86. data/lib/dynflow/testing/factories.rb +83 -0
  87. data/lib/dynflow/testing/managed_clock.rb +23 -0
  88. data/lib/dynflow/testing/mimic.rb +38 -0
  89. data/lib/dynflow/transaction_adapters.rb +9 -0
  90. data/lib/dynflow/transaction_adapters/abstract.rb +26 -0
  91. data/lib/dynflow/transaction_adapters/active_record.rb +27 -0
  92. data/lib/dynflow/transaction_adapters/none.rb +12 -0
  93. data/lib/dynflow/version.rb +1 -1
  94. data/lib/dynflow/web_console.rb +277 -0
  95. data/lib/dynflow/world.rb +168 -0
  96. data/test/action_test.rb +89 -11
  97. data/test/clock_test.rb +59 -0
  98. data/test/code_workflow_example.rb +382 -0
  99. data/test/execution_plan_test.rb +195 -64
  100. data/test/executor_test.rb +692 -0
  101. data/test/persistance_adapters_test.rb +173 -0
  102. data/test/test_helper.rb +316 -1
  103. data/test/testing_test.rb +148 -0
  104. data/test/web_console_test.rb +38 -0
  105. data/web/assets/javascripts/application.js +25 -0
  106. data/web/assets/stylesheets/application.css +101 -0
  107. data/web/assets/vendor/bootstrap/css/bootstrap-responsive.css +1109 -0
  108. data/web/assets/vendor/bootstrap/css/bootstrap-responsive.min.css +9 -0
  109. data/web/assets/vendor/bootstrap/css/bootstrap.css +6167 -0
  110. data/web/assets/vendor/bootstrap/css/bootstrap.min.css +9 -0
  111. data/web/assets/vendor/bootstrap/img/glyphicons-halflings-white.png +0 -0
  112. data/web/assets/vendor/bootstrap/img/glyphicons-halflings.png +0 -0
  113. data/web/assets/vendor/bootstrap/js/bootstrap.js +2280 -0
  114. data/web/assets/vendor/bootstrap/js/bootstrap.min.js +6 -0
  115. data/web/assets/vendor/google-code-prettify/lang-basic.js +3 -0
  116. data/web/assets/vendor/google-code-prettify/prettify.css +1 -0
  117. data/web/assets/vendor/google-code-prettify/prettify.js +30 -0
  118. data/web/assets/vendor/google-code-prettify/run_prettify.js +34 -0
  119. data/web/assets/vendor/jquery/jquery.js +9807 -0
  120. data/web/views/flow.erb +19 -0
  121. data/web/views/flow_step.erb +31 -0
  122. data/web/views/index.erb +39 -0
  123. data/web/views/layout.erb +20 -0
  124. data/web/views/plan_step.erb +11 -0
  125. data/web/views/show.erb +54 -0
  126. metadata +250 -11
  127. data/examples/events.rb +0 -71
  128. data/examples/workflow.rb +0 -140
  129. data/lib/dynflow/bus.rb +0 -168
  130. data/lib/dynflow/dispatcher.rb +0 -36
  131. data/lib/dynflow/logger.rb +0 -34
  132. data/lib/dynflow/step.rb +0 -234
  133. data/test/bus_test.rb +0 -150
data/lib/dynflow/step.rb DELETED
@@ -1,234 +0,0 @@
1
- module Dynflow
2
- class Step
3
-
4
- class Reference
5
-
6
- attr_reader :step, :field
7
-
8
- def initialize(step, field)
9
- unless %w[input output].include? field
10
- raise "Unexpected reference field: #{field}. Only input and output allowed"
11
- end
12
- @step = step
13
- @field = field
14
- end
15
-
16
- def encode
17
- unless @step.persistence
18
- raise "Reference can't be serialized without persistence available"
19
- end
20
-
21
- {
22
- 'dynflow_step_persistence_id' => @step.persistence.persistence_id,
23
- 'field' => @field
24
- }
25
- end
26
-
27
- def self.decode(data)
28
- return nil unless data.is_a? Hash
29
- return nil unless data.has_key?('dynflow_step_persistence_id')
30
- persistence_id = data['dynflow_step_persistence_id']
31
- self.new(Dynflow::Bus.persisted_step(persistence_id), data['field'])
32
- end
33
-
34
- def dereference
35
- @step.send(@field)
36
- end
37
-
38
- def inspect
39
- ret = "References "
40
- ret << @step.class.name.split('::').last
41
- ret << "/"
42
- ret << @step.action_class.name
43
- ret << "(#{@step.persistence.persistence_id})" if @step.persistence
44
- ret << "/"
45
- ret << @field
46
- return ret
47
- end
48
-
49
- end
50
-
51
- extend Forwardable
52
-
53
- def_delegators :@data, '[]', '[]='
54
-
55
- attr_accessor :status
56
- attr_reader :data, :action_class
57
-
58
- # persistent representation of the step
59
- attr_accessor :persistence
60
-
61
- def input
62
- @data['input']
63
- end
64
-
65
- def input=(input)
66
- @data['input'] = input
67
- end
68
-
69
- def output
70
- @data['output']
71
- end
72
-
73
- def output=(output)
74
- @data['output'] = output
75
- end
76
-
77
- def error
78
- @data['error']
79
- end
80
-
81
- def error=(error)
82
- @data['error'] = error
83
- end
84
-
85
- # get a fresh instance of action class for execution
86
- def action
87
- self.action_class.new(input, output)
88
- end
89
-
90
- def catch_errors
91
- yield
92
- self.status = 'success'
93
- return true
94
- rescue Exception => e
95
- self.error = {
96
- "exception" => e.class.name,
97
- "message" => e.message,
98
- "backtrace" => e.backtrace
99
- }
100
- self.status = 'error'
101
- return false
102
- end
103
-
104
- def ==(other)
105
- self.encode == other.encode
106
- end
107
-
108
- def self.decode(data)
109
- ret = data['step_class'].constantize.allocate
110
- ret.instance_variable_set("@action_class", data['action_class'].constantize)
111
- ret.instance_variable_set("@status", data['status'])
112
- ret.instance_variable_set("@data", decode_data(data['data']))
113
- return ret
114
- end
115
-
116
- def encode
117
- {
118
- 'step_class' => self.class.name,
119
- 'action_class' => action_class.name,
120
- 'status' => status,
121
- 'data' => encoded_data
122
- }
123
- end
124
-
125
- def self.decode_data(data)
126
- walk(data) do |item|
127
- Reference.decode(item)
128
- end
129
- end
130
-
131
- # we need this to encode the reference correctly
132
- def encoded_data
133
- self.class.walk(data) do |item|
134
- if item.is_a? Reference
135
- item.encode
136
- end
137
- end
138
- end
139
-
140
- def replace_references!
141
- @data = self.class.walk(data) do |item|
142
- if item.is_a? Reference
143
- if item.step.status == 'skipped' || item.step.status == 'error'
144
- self.status = 'skipped'
145
- item
146
- else
147
- item.dereference
148
- end
149
- end
150
- end
151
- end
152
-
153
- # walks hash depth-first, yielding on every value
154
- # if yield return non-false value, use that instead of original
155
- # value in a resulting hash
156
- def self.walk(data, &block)
157
- if converted = (yield data)
158
- return converted
159
- end
160
- case data
161
- when Array
162
- data.map { |d| walk(d, &block) }
163
- when Hash
164
- data.reduce({}) { |h, (k, v)| h.update(k => walk(v, &block)) }
165
- else
166
- data
167
- end
168
- end
169
-
170
- def persist
171
- if @persistence
172
- @persistence.persist(self)
173
- end
174
- end
175
-
176
- def persist_before_run
177
- if @persistence
178
- @persistence.before_run(self)
179
- end
180
- end
181
-
182
- def persist_after_run
183
- if @persistence
184
- @persistence.after_run(self)
185
- end
186
- end
187
-
188
- class Plan < Step
189
-
190
- def initialize(action)
191
- # we want to have the steps separated:
192
- # not using the original action object
193
- @action_class = action.class
194
- self.status = 'finished' # default status
195
- @data = {}.with_indifferent_access
196
- end
197
-
198
- end
199
-
200
- class Run < Step
201
-
202
- def initialize(action)
203
- # we want to have the steps separated:
204
- # not using the original action object
205
- @action_class = action.class
206
- self.status = 'pending' # default status
207
- @data = {
208
- 'input' => action.input,
209
- 'output' => action.output
210
- }.with_indifferent_access
211
- end
212
-
213
- end
214
-
215
- class Finalize < Step
216
-
217
- def initialize(run_step)
218
- # we want to have the steps separated:
219
- # not using the original action object
220
- @action_class = run_step.action_class
221
- self.status = 'pending' # default status
222
- if run_step.action.respond_to? :run
223
- @data = {
224
- 'input' => Reference.new(run_step, 'input'),
225
- 'output' => Reference.new(run_step, 'output'),
226
- }
227
- else
228
- @data = run_step.data
229
- end
230
- end
231
-
232
- end
233
- end
234
- end
data/test/bus_test.rb DELETED
@@ -1,150 +0,0 @@
1
- require 'test_helper'
2
- require 'set'
3
-
4
- module Dynflow
5
- module BusTest
6
- describe "bus" do
7
-
8
- class Promotion < Action
9
-
10
- def plan(repo_names, package_names)
11
- repo_names.each do |repo_name|
12
- plan_action(CloneRepo, {'name' => repo_name})
13
- end
14
-
15
- package_names.each do |package_name|
16
- plan_action(ClonePackage, {'name' => package_name})
17
- end
18
- end
19
-
20
- end
21
-
22
- class CloneRepo < Action
23
-
24
- input_format do
25
- param :name, String
26
- end
27
-
28
- output_format do
29
- param :id, String
30
- end
31
-
32
- def plan(input)
33
- raise 'Simulate error in plan phase' if input['name'] == 'fail_in_plan'
34
- plan_self(input)
35
- end
36
-
37
- def run
38
- raise 'Simulate error in execution phase' if input['name'] == 'fail_in_run'
39
- output['id'] = input['name']
40
- end
41
-
42
- def finalize(outputs)
43
- raise 'Simulate error in finalize phase' if input['name'] == 'fail_in_finalize'
44
- end
45
-
46
- end
47
-
48
- it 'returns the execution plan obejct when triggering an action' do
49
- Promotion.trigger(['sucess'], []).must_be_instance_of Dynflow::ExecutionPlan
50
- end
51
-
52
- describe 'handling errros in plan phase' do
53
-
54
- let(:failed_plan) { Promotion.trigger(['fail_in_plan'], []) }
55
- let(:failed_step) { failed_plan.plan_steps.last }
56
-
57
- it 'marks the process as error' do
58
- failed_plan.status.must_equal 'error'
59
- end
60
-
61
- it 'saves errors of actions' do
62
- failed_step.status.must_equal "error"
63
- expected_error = {
64
- 'exception' => 'RuntimeError',
65
- 'message' => 'Simulate error in plan phase'
66
- }
67
- failed_step.error['exception'].must_equal expected_error['exception']
68
- failed_step.error['message'].must_equal expected_error['message']
69
- failed_step.error['backtrace'].must_be_instance_of Array
70
- end
71
-
72
- end
73
-
74
- describe 'handling errros in execution phase' do
75
-
76
- let(:failed_plan) { Promotion.trigger(['fail_in_run'], []) }
77
- let(:failed_step) { failed_plan.run_steps.first }
78
-
79
- it 'pauses the process' do
80
- failed_plan.status.must_equal 'paused'
81
- end
82
-
83
- it 'saves errors of actions' do
84
- failed_step.status.must_equal "error"
85
- expected_error = {
86
- 'exception' => 'RuntimeError',
87
- 'message' => 'Simulate error in execution phase'
88
- }
89
- failed_step.error['exception'].must_equal expected_error['exception']
90
- failed_step.error['message'].must_equal expected_error['message']
91
- failed_step.error['backtrace'].must_be_instance_of Array
92
- end
93
-
94
- it 'allows skipping the step' do
95
- Dynflow::Bus.skip(failed_step)
96
- Dynflow::Bus.resume(failed_plan)
97
-
98
- failed_plan.status.must_equal 'finished'
99
- failed_step.status.must_equal 'skipped'
100
- end
101
-
102
- it 'allows rerunning the step' do
103
- failed_step.input['name'] = 'succeed'
104
- Dynflow::Bus.resume(failed_plan)
105
-
106
- failed_plan.status.must_equal 'finished'
107
- failed_step.output.must_equal('id' => 'succeed')
108
- end
109
-
110
- end
111
-
112
- describe 'handling errors in finalizatoin phase' do
113
-
114
- let(:failed_plan) { Promotion.trigger(['fail_in_finalize'], []) }
115
- let(:failed_step) { failed_plan.finalize_steps.first }
116
-
117
- it 'pauses the process' do
118
- failed_plan.status.must_equal 'paused'
119
- end
120
-
121
- it 'saves errors of actions' do
122
- expected_error = {
123
- 'exception' => 'RuntimeError',
124
- 'message' => 'Simulate error in finalize phase'
125
- }
126
- failed_step.error['exception'].must_equal expected_error['exception']
127
- failed_step.error['message'].must_equal expected_error['message']
128
- failed_step.error['backtrace'].must_be_instance_of Array
129
- end
130
-
131
- it 'allows finishing a finalize phase' do
132
- failed_step.input['name'] = 'succeed'
133
- Dynflow::Bus.resume(failed_plan)
134
-
135
- failed_plan.status.must_equal 'finished'
136
- end
137
-
138
- it 'allows skipping the step' do
139
- Dynflow::Bus.skip(failed_step)
140
- Dynflow::Bus.resume(failed_plan)
141
-
142
- failed_plan.status.must_equal 'finished'
143
- failed_step.status.must_equal 'skipped'
144
- end
145
-
146
- end
147
-
148
- end
149
- end
150
- end