dynflow 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. data/.gitignore +6 -0
  2. data/.travis.yml +9 -0
  3. data/Gemfile +0 -10
  4. data/MIT-LICENSE +1 -1
  5. data/README.md +99 -37
  6. data/Rakefile +2 -6
  7. data/doc/images/logo.png +0 -0
  8. data/dynflow.gemspec +10 -1
  9. data/examples/generate_work_for_daemon.rb +24 -0
  10. data/examples/orchestrate.rb +121 -0
  11. data/examples/run_daemon.rb +17 -0
  12. data/examples/web_console.rb +29 -0
  13. data/lib/dynflow.rb +27 -6
  14. data/lib/dynflow/action.rb +185 -77
  15. data/lib/dynflow/action/cancellable_polling.rb +18 -0
  16. data/lib/dynflow/action/finalize_phase.rb +18 -0
  17. data/lib/dynflow/action/flow_phase.rb +44 -0
  18. data/lib/dynflow/action/format.rb +46 -0
  19. data/lib/dynflow/action/missing.rb +26 -0
  20. data/lib/dynflow/action/plan_phase.rb +85 -0
  21. data/lib/dynflow/action/polling.rb +49 -0
  22. data/lib/dynflow/action/presenter.rb +51 -0
  23. data/lib/dynflow/action/progress.rb +62 -0
  24. data/lib/dynflow/action/run_phase.rb +43 -0
  25. data/lib/dynflow/action/suspended.rb +21 -0
  26. data/lib/dynflow/clock.rb +133 -0
  27. data/lib/dynflow/daemon.rb +29 -0
  28. data/lib/dynflow/execution_plan.rb +285 -33
  29. data/lib/dynflow/execution_plan/dependency_graph.rb +29 -0
  30. data/lib/dynflow/execution_plan/output_reference.rb +52 -0
  31. data/lib/dynflow/execution_plan/steps.rb +12 -0
  32. data/lib/dynflow/execution_plan/steps/abstract.rb +121 -0
  33. data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb +52 -0
  34. data/lib/dynflow/execution_plan/steps/error.rb +33 -0
  35. data/lib/dynflow/execution_plan/steps/finalize_step.rb +23 -0
  36. data/lib/dynflow/execution_plan/steps/plan_step.rb +81 -0
  37. data/lib/dynflow/execution_plan/steps/run_step.rb +21 -0
  38. data/lib/dynflow/executors.rb +9 -0
  39. data/lib/dynflow/executors/abstract.rb +32 -0
  40. data/lib/dynflow/executors/parallel.rb +88 -0
  41. data/lib/dynflow/executors/parallel/core.rb +119 -0
  42. data/lib/dynflow/executors/parallel/execution_plan_manager.rb +120 -0
  43. data/lib/dynflow/executors/parallel/flow_manager.rb +48 -0
  44. data/lib/dynflow/executors/parallel/pool.rb +102 -0
  45. data/lib/dynflow/executors/parallel/running_steps_manager.rb +63 -0
  46. data/lib/dynflow/executors/parallel/sequence_cursor.rb +97 -0
  47. data/lib/dynflow/executors/parallel/sequential_manager.rb +81 -0
  48. data/lib/dynflow/executors/parallel/work_queue.rb +44 -0
  49. data/lib/dynflow/executors/parallel/worker.rb +30 -0
  50. data/lib/dynflow/executors/remote_via_socket.rb +38 -0
  51. data/lib/dynflow/executors/remote_via_socket/core.rb +150 -0
  52. data/lib/dynflow/flows.rb +13 -0
  53. data/lib/dynflow/flows/abstract.rb +36 -0
  54. data/lib/dynflow/flows/abstract_composed.rb +104 -0
  55. data/lib/dynflow/flows/atom.rb +36 -0
  56. data/lib/dynflow/flows/concurrence.rb +28 -0
  57. data/lib/dynflow/flows/sequence.rb +13 -0
  58. data/lib/dynflow/future.rb +173 -0
  59. data/lib/dynflow/listeners.rb +7 -0
  60. data/lib/dynflow/listeners/abstract.rb +13 -0
  61. data/lib/dynflow/listeners/serialization.rb +41 -0
  62. data/lib/dynflow/listeners/socket.rb +88 -0
  63. data/lib/dynflow/logger_adapters.rb +8 -0
  64. data/lib/dynflow/logger_adapters/abstract.rb +30 -0
  65. data/lib/dynflow/logger_adapters/delegator.rb +13 -0
  66. data/lib/dynflow/logger_adapters/formatters.rb +8 -0
  67. data/lib/dynflow/logger_adapters/formatters/abstract.rb +33 -0
  68. data/lib/dynflow/logger_adapters/formatters/exception.rb +15 -0
  69. data/lib/dynflow/logger_adapters/simple.rb +59 -0
  70. data/lib/dynflow/micro_actor.rb +102 -0
  71. data/lib/dynflow/persistence.rb +53 -0
  72. data/lib/dynflow/persistence_adapters.rb +6 -0
  73. data/lib/dynflow/persistence_adapters/abstract.rb +56 -0
  74. data/lib/dynflow/persistence_adapters/sequel.rb +160 -0
  75. data/lib/dynflow/persistence_adapters/sequel_migrations/001_initial.rb +52 -0
  76. data/lib/dynflow/serializable.rb +66 -0
  77. data/lib/dynflow/simple_world.rb +18 -0
  78. data/lib/dynflow/stateful.rb +40 -0
  79. data/lib/dynflow/testing.rb +32 -0
  80. data/lib/dynflow/testing/assertions.rb +64 -0
  81. data/lib/dynflow/testing/dummy_execution_plan.rb +40 -0
  82. data/lib/dynflow/testing/dummy_executor.rb +29 -0
  83. data/lib/dynflow/testing/dummy_planned_action.rb +18 -0
  84. data/lib/dynflow/testing/dummy_step.rb +19 -0
  85. data/lib/dynflow/testing/dummy_world.rb +33 -0
  86. data/lib/dynflow/testing/factories.rb +83 -0
  87. data/lib/dynflow/testing/managed_clock.rb +23 -0
  88. data/lib/dynflow/testing/mimic.rb +38 -0
  89. data/lib/dynflow/transaction_adapters.rb +9 -0
  90. data/lib/dynflow/transaction_adapters/abstract.rb +26 -0
  91. data/lib/dynflow/transaction_adapters/active_record.rb +27 -0
  92. data/lib/dynflow/transaction_adapters/none.rb +12 -0
  93. data/lib/dynflow/version.rb +1 -1
  94. data/lib/dynflow/web_console.rb +277 -0
  95. data/lib/dynflow/world.rb +168 -0
  96. data/test/action_test.rb +89 -11
  97. data/test/clock_test.rb +59 -0
  98. data/test/code_workflow_example.rb +382 -0
  99. data/test/execution_plan_test.rb +195 -64
  100. data/test/executor_test.rb +692 -0
  101. data/test/persistance_adapters_test.rb +173 -0
  102. data/test/test_helper.rb +316 -1
  103. data/test/testing_test.rb +148 -0
  104. data/test/web_console_test.rb +38 -0
  105. data/web/assets/javascripts/application.js +25 -0
  106. data/web/assets/stylesheets/application.css +101 -0
  107. data/web/assets/vendor/bootstrap/css/bootstrap-responsive.css +1109 -0
  108. data/web/assets/vendor/bootstrap/css/bootstrap-responsive.min.css +9 -0
  109. data/web/assets/vendor/bootstrap/css/bootstrap.css +6167 -0
  110. data/web/assets/vendor/bootstrap/css/bootstrap.min.css +9 -0
  111. data/web/assets/vendor/bootstrap/img/glyphicons-halflings-white.png +0 -0
  112. data/web/assets/vendor/bootstrap/img/glyphicons-halflings.png +0 -0
  113. data/web/assets/vendor/bootstrap/js/bootstrap.js +2280 -0
  114. data/web/assets/vendor/bootstrap/js/bootstrap.min.js +6 -0
  115. data/web/assets/vendor/google-code-prettify/lang-basic.js +3 -0
  116. data/web/assets/vendor/google-code-prettify/prettify.css +1 -0
  117. data/web/assets/vendor/google-code-prettify/prettify.js +30 -0
  118. data/web/assets/vendor/google-code-prettify/run_prettify.js +34 -0
  119. data/web/assets/vendor/jquery/jquery.js +9807 -0
  120. data/web/views/flow.erb +19 -0
  121. data/web/views/flow_step.erb +31 -0
  122. data/web/views/index.erb +39 -0
  123. data/web/views/layout.erb +20 -0
  124. data/web/views/plan_step.erb +11 -0
  125. data/web/views/show.erb +54 -0
  126. metadata +250 -11
  127. data/examples/events.rb +0 -71
  128. data/examples/workflow.rb +0 -140
  129. data/lib/dynflow/bus.rb +0 -168
  130. data/lib/dynflow/dispatcher.rb +0 -36
  131. data/lib/dynflow/logger.rb +0 -34
  132. data/lib/dynflow/step.rb +0 -234
  133. data/test/bus_test.rb +0 -150
data/lib/dynflow/step.rb DELETED
@@ -1,234 +0,0 @@
1
- module Dynflow
2
- class Step
3
-
4
- class Reference
5
-
6
- attr_reader :step, :field
7
-
8
- def initialize(step, field)
9
- unless %w[input output].include? field
10
- raise "Unexpected reference field: #{field}. Only input and output allowed"
11
- end
12
- @step = step
13
- @field = field
14
- end
15
-
16
- def encode
17
- unless @step.persistence
18
- raise "Reference can't be serialized without persistence available"
19
- end
20
-
21
- {
22
- 'dynflow_step_persistence_id' => @step.persistence.persistence_id,
23
- 'field' => @field
24
- }
25
- end
26
-
27
- def self.decode(data)
28
- return nil unless data.is_a? Hash
29
- return nil unless data.has_key?('dynflow_step_persistence_id')
30
- persistence_id = data['dynflow_step_persistence_id']
31
- self.new(Dynflow::Bus.persisted_step(persistence_id), data['field'])
32
- end
33
-
34
- def dereference
35
- @step.send(@field)
36
- end
37
-
38
- def inspect
39
- ret = "References "
40
- ret << @step.class.name.split('::').last
41
- ret << "/"
42
- ret << @step.action_class.name
43
- ret << "(#{@step.persistence.persistence_id})" if @step.persistence
44
- ret << "/"
45
- ret << @field
46
- return ret
47
- end
48
-
49
- end
50
-
51
- extend Forwardable
52
-
53
- def_delegators :@data, '[]', '[]='
54
-
55
- attr_accessor :status
56
- attr_reader :data, :action_class
57
-
58
- # persistent representation of the step
59
- attr_accessor :persistence
60
-
61
- def input
62
- @data['input']
63
- end
64
-
65
- def input=(input)
66
- @data['input'] = input
67
- end
68
-
69
- def output
70
- @data['output']
71
- end
72
-
73
- def output=(output)
74
- @data['output'] = output
75
- end
76
-
77
- def error
78
- @data['error']
79
- end
80
-
81
- def error=(error)
82
- @data['error'] = error
83
- end
84
-
85
- # get a fresh instance of action class for execution
86
- def action
87
- self.action_class.new(input, output)
88
- end
89
-
90
- def catch_errors
91
- yield
92
- self.status = 'success'
93
- return true
94
- rescue Exception => e
95
- self.error = {
96
- "exception" => e.class.name,
97
- "message" => e.message,
98
- "backtrace" => e.backtrace
99
- }
100
- self.status = 'error'
101
- return false
102
- end
103
-
104
- def ==(other)
105
- self.encode == other.encode
106
- end
107
-
108
- def self.decode(data)
109
- ret = data['step_class'].constantize.allocate
110
- ret.instance_variable_set("@action_class", data['action_class'].constantize)
111
- ret.instance_variable_set("@status", data['status'])
112
- ret.instance_variable_set("@data", decode_data(data['data']))
113
- return ret
114
- end
115
-
116
- def encode
117
- {
118
- 'step_class' => self.class.name,
119
- 'action_class' => action_class.name,
120
- 'status' => status,
121
- 'data' => encoded_data
122
- }
123
- end
124
-
125
- def self.decode_data(data)
126
- walk(data) do |item|
127
- Reference.decode(item)
128
- end
129
- end
130
-
131
- # we need this to encode the reference correctly
132
- def encoded_data
133
- self.class.walk(data) do |item|
134
- if item.is_a? Reference
135
- item.encode
136
- end
137
- end
138
- end
139
-
140
- def replace_references!
141
- @data = self.class.walk(data) do |item|
142
- if item.is_a? Reference
143
- if item.step.status == 'skipped' || item.step.status == 'error'
144
- self.status = 'skipped'
145
- item
146
- else
147
- item.dereference
148
- end
149
- end
150
- end
151
- end
152
-
153
- # walks hash depth-first, yielding on every value
154
- # if yield return non-false value, use that instead of original
155
- # value in a resulting hash
156
- def self.walk(data, &block)
157
- if converted = (yield data)
158
- return converted
159
- end
160
- case data
161
- when Array
162
- data.map { |d| walk(d, &block) }
163
- when Hash
164
- data.reduce({}) { |h, (k, v)| h.update(k => walk(v, &block)) }
165
- else
166
- data
167
- end
168
- end
169
-
170
- def persist
171
- if @persistence
172
- @persistence.persist(self)
173
- end
174
- end
175
-
176
- def persist_before_run
177
- if @persistence
178
- @persistence.before_run(self)
179
- end
180
- end
181
-
182
- def persist_after_run
183
- if @persistence
184
- @persistence.after_run(self)
185
- end
186
- end
187
-
188
- class Plan < Step
189
-
190
- def initialize(action)
191
- # we want to have the steps separated:
192
- # not using the original action object
193
- @action_class = action.class
194
- self.status = 'finished' # default status
195
- @data = {}.with_indifferent_access
196
- end
197
-
198
- end
199
-
200
- class Run < Step
201
-
202
- def initialize(action)
203
- # we want to have the steps separated:
204
- # not using the original action object
205
- @action_class = action.class
206
- self.status = 'pending' # default status
207
- @data = {
208
- 'input' => action.input,
209
- 'output' => action.output
210
- }.with_indifferent_access
211
- end
212
-
213
- end
214
-
215
- class Finalize < Step
216
-
217
- def initialize(run_step)
218
- # we want to have the steps separated:
219
- # not using the original action object
220
- @action_class = run_step.action_class
221
- self.status = 'pending' # default status
222
- if run_step.action.respond_to? :run
223
- @data = {
224
- 'input' => Reference.new(run_step, 'input'),
225
- 'output' => Reference.new(run_step, 'output'),
226
- }
227
- else
228
- @data = run_step.data
229
- end
230
- end
231
-
232
- end
233
- end
234
- end
data/test/bus_test.rb DELETED
@@ -1,150 +0,0 @@
1
- require 'test_helper'
2
- require 'set'
3
-
4
- module Dynflow
5
- module BusTest
6
- describe "bus" do
7
-
8
- class Promotion < Action
9
-
10
- def plan(repo_names, package_names)
11
- repo_names.each do |repo_name|
12
- plan_action(CloneRepo, {'name' => repo_name})
13
- end
14
-
15
- package_names.each do |package_name|
16
- plan_action(ClonePackage, {'name' => package_name})
17
- end
18
- end
19
-
20
- end
21
-
22
- class CloneRepo < Action
23
-
24
- input_format do
25
- param :name, String
26
- end
27
-
28
- output_format do
29
- param :id, String
30
- end
31
-
32
- def plan(input)
33
- raise 'Simulate error in plan phase' if input['name'] == 'fail_in_plan'
34
- plan_self(input)
35
- end
36
-
37
- def run
38
- raise 'Simulate error in execution phase' if input['name'] == 'fail_in_run'
39
- output['id'] = input['name']
40
- end
41
-
42
- def finalize(outputs)
43
- raise 'Simulate error in finalize phase' if input['name'] == 'fail_in_finalize'
44
- end
45
-
46
- end
47
-
48
- it 'returns the execution plan obejct when triggering an action' do
49
- Promotion.trigger(['sucess'], []).must_be_instance_of Dynflow::ExecutionPlan
50
- end
51
-
52
- describe 'handling errros in plan phase' do
53
-
54
- let(:failed_plan) { Promotion.trigger(['fail_in_plan'], []) }
55
- let(:failed_step) { failed_plan.plan_steps.last }
56
-
57
- it 'marks the process as error' do
58
- failed_plan.status.must_equal 'error'
59
- end
60
-
61
- it 'saves errors of actions' do
62
- failed_step.status.must_equal "error"
63
- expected_error = {
64
- 'exception' => 'RuntimeError',
65
- 'message' => 'Simulate error in plan phase'
66
- }
67
- failed_step.error['exception'].must_equal expected_error['exception']
68
- failed_step.error['message'].must_equal expected_error['message']
69
- failed_step.error['backtrace'].must_be_instance_of Array
70
- end
71
-
72
- end
73
-
74
- describe 'handling errros in execution phase' do
75
-
76
- let(:failed_plan) { Promotion.trigger(['fail_in_run'], []) }
77
- let(:failed_step) { failed_plan.run_steps.first }
78
-
79
- it 'pauses the process' do
80
- failed_plan.status.must_equal 'paused'
81
- end
82
-
83
- it 'saves errors of actions' do
84
- failed_step.status.must_equal "error"
85
- expected_error = {
86
- 'exception' => 'RuntimeError',
87
- 'message' => 'Simulate error in execution phase'
88
- }
89
- failed_step.error['exception'].must_equal expected_error['exception']
90
- failed_step.error['message'].must_equal expected_error['message']
91
- failed_step.error['backtrace'].must_be_instance_of Array
92
- end
93
-
94
- it 'allows skipping the step' do
95
- Dynflow::Bus.skip(failed_step)
96
- Dynflow::Bus.resume(failed_plan)
97
-
98
- failed_plan.status.must_equal 'finished'
99
- failed_step.status.must_equal 'skipped'
100
- end
101
-
102
- it 'allows rerunning the step' do
103
- failed_step.input['name'] = 'succeed'
104
- Dynflow::Bus.resume(failed_plan)
105
-
106
- failed_plan.status.must_equal 'finished'
107
- failed_step.output.must_equal('id' => 'succeed')
108
- end
109
-
110
- end
111
-
112
- describe 'handling errors in finalizatoin phase' do
113
-
114
- let(:failed_plan) { Promotion.trigger(['fail_in_finalize'], []) }
115
- let(:failed_step) { failed_plan.finalize_steps.first }
116
-
117
- it 'pauses the process' do
118
- failed_plan.status.must_equal 'paused'
119
- end
120
-
121
- it 'saves errors of actions' do
122
- expected_error = {
123
- 'exception' => 'RuntimeError',
124
- 'message' => 'Simulate error in finalize phase'
125
- }
126
- failed_step.error['exception'].must_equal expected_error['exception']
127
- failed_step.error['message'].must_equal expected_error['message']
128
- failed_step.error['backtrace'].must_be_instance_of Array
129
- end
130
-
131
- it 'allows finishing a finalize phase' do
132
- failed_step.input['name'] = 'succeed'
133
- Dynflow::Bus.resume(failed_plan)
134
-
135
- failed_plan.status.must_equal 'finished'
136
- end
137
-
138
- it 'allows skipping the step' do
139
- Dynflow::Bus.skip(failed_step)
140
- Dynflow::Bus.resume(failed_plan)
141
-
142
- failed_plan.status.must_equal 'finished'
143
- failed_step.status.must_equal 'skipped'
144
- end
145
-
146
- end
147
-
148
- end
149
- end
150
- end