dynflow 0.2.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. data/.travis.yml +1 -0
  2. data/lib/dynflow.rb +1 -0
  3. data/lib/dynflow/action.rb +5 -3
  4. data/lib/dynflow/action/finalize_phase.rb +3 -1
  5. data/lib/dynflow/action/plan_phase.rb +4 -2
  6. data/lib/dynflow/action/run_phase.rb +3 -1
  7. data/lib/dynflow/daemon.rb +1 -0
  8. data/lib/dynflow/execution_plan.rb +6 -4
  9. data/lib/dynflow/executors/abstract.rb +12 -0
  10. data/lib/dynflow/executors/parallel.rb +16 -35
  11. data/lib/dynflow/executors/parallel/core.rb +13 -8
  12. data/lib/dynflow/executors/parallel/execution_plan_manager.rb +21 -29
  13. data/lib/dynflow/executors/parallel/flow_manager.rb +0 -3
  14. data/lib/dynflow/executors/parallel/running_steps_manager.rb +5 -3
  15. data/lib/dynflow/executors/parallel/sequential_manager.rb +6 -2
  16. data/lib/dynflow/executors/parallel/worker.rb +5 -4
  17. data/lib/dynflow/executors/remote_via_socket.rb +7 -2
  18. data/lib/dynflow/executors/remote_via_socket/core.rb +66 -32
  19. data/lib/dynflow/future.rb +1 -1
  20. data/lib/dynflow/listeners/abstract.rb +4 -0
  21. data/lib/dynflow/listeners/serialization.rb +42 -8
  22. data/lib/dynflow/listeners/socket.rb +49 -19
  23. data/lib/dynflow/middleware.rb +46 -0
  24. data/lib/dynflow/middleware/action.rb +9 -0
  25. data/lib/dynflow/middleware/register.rb +32 -0
  26. data/lib/dynflow/middleware/resolver.rb +63 -0
  27. data/lib/dynflow/middleware/stack.rb +29 -0
  28. data/lib/dynflow/middleware/world.rb +58 -0
  29. data/lib/dynflow/simple_world.rb +1 -0
  30. data/lib/dynflow/testing/dummy_world.rb +3 -1
  31. data/lib/dynflow/version.rb +1 -1
  32. data/lib/dynflow/web_console.rb +7 -2
  33. data/lib/dynflow/world.rb +29 -9
  34. data/test/action_test.rb +5 -6
  35. data/test/execution_plan_test.rb +10 -11
  36. data/test/executor_test.rb +152 -89
  37. data/test/middleware_test.rb +109 -0
  38. data/test/remote_via_socket_test.rb +166 -0
  39. data/test/{code_workflow_example.rb → support/code_workflow_example.rb} +39 -30
  40. data/test/support/middleware_example.rb +132 -0
  41. data/test/test_helper.rb +18 -16
  42. data/test/testing_test.rb +4 -3
  43. data/test/web_console_test.rb +1 -2
  44. metadata +16 -4
data/.travis.yml CHANGED
@@ -4,6 +4,7 @@ language:
4
4
  rvm:
5
5
  - "1.9.3"
6
6
  - "2.0.0"
7
+ - "2.1.0"
7
8
 
8
9
  script:
9
10
  - bundle exec rake test
data/lib/dynflow.rb CHANGED
@@ -19,6 +19,7 @@ module Dynflow
19
19
  require 'dynflow/stateful'
20
20
  require 'dynflow/transaction_adapters'
21
21
  require 'dynflow/persistence'
22
+ require 'dynflow/middleware'
22
23
  require 'dynflow/action'
23
24
  require 'dynflow/flows'
24
25
  require 'dynflow/execution_plan'
@@ -10,6 +10,8 @@ module Dynflow
10
10
  require 'dynflow/action/format'
11
11
  extend Format
12
12
 
13
+ extend Middleware::Action
14
+
13
15
  require 'dynflow/action/progress'
14
16
  include Progress
15
17
 
@@ -188,19 +190,19 @@ module Dynflow
188
190
 
189
191
  private
190
192
 
191
- ERRORING = Object.new
193
+ ERROR = Object.new
192
194
 
193
195
  # DSL to terminate action execution and set it to error
194
196
  def error!(error)
195
197
  set_error(error)
196
- throw ERRORING
198
+ throw ERROR
197
199
  end
198
200
 
199
201
  def with_error_handling(&block)
200
202
  raise "wrong state #{self.state}" unless self.state == :running
201
203
 
202
204
  begin
203
- catch(ERRORING) { block.call }
205
+ catch(ERROR) { block.call }
204
206
  rescue Exception => error
205
207
  set_error(error)
206
208
  # reraise low-level exceptions
@@ -10,7 +10,9 @@ module Dynflow
10
10
  self.state = :running
11
11
  save_state
12
12
  with_error_handling do
13
- finalize
13
+ world.middleware.execute(:finalize, self) do
14
+ finalize
15
+ end
14
16
  end
15
17
  end
16
18
 
@@ -20,8 +20,10 @@ module Dynflow
20
20
  self.state = :running
21
21
  save_state
22
22
  with_error_handling do
23
- execution_plan.switch_flow(Flows::Concurrence.new([])) do
24
- plan(*args)
23
+ concurrence do
24
+ world.middleware.execute(:plan, self, *args) do |*new_args|
25
+ plan(*new_args)
26
+ end
25
27
  end
26
28
 
27
29
  subscribed_actions = world.subscribed_actions(self.action_class)
@@ -22,7 +22,9 @@ module Dynflow
22
22
  self.state = :running
23
23
  save_state
24
24
  with_error_handling do
25
- result = catch(SUSPEND) { event ? run(event) : run }
25
+ result = catch(SUSPEND) do
26
+ world.middleware.execute(:run, self, *Array(event)) { |*args| run(*args) }
27
+ end
26
28
  if result == SUSPEND
27
29
  self.state = :suspended
28
30
  end
@@ -13,6 +13,7 @@ module Dynflow
13
13
  terminated = Future.new
14
14
  trap('SIGINT') { @world.terminate terminated }
15
15
  terminated.wait
16
+ @listener.terminate.wait
16
17
  end
17
18
  end
18
19
 
@@ -114,11 +114,13 @@ module Dynflow
114
114
  def plan(*args)
115
115
  update_state(:planning)
116
116
  world.transaction_adapter.transaction do
117
- with_planning_scope do
118
- root_plan_step.execute(self, nil, *args)
117
+ world.middleware.execute(:plan_phase, root_plan_step.action_class) do
118
+ with_planning_scope do
119
+ root_plan_step.execute(self, nil, *args)
119
120
 
120
- if @dependency_graph.unresolved?
121
- raise "Some dependencies were not resolved: #{@dependency_graph.inspect}"
121
+ if @dependency_graph.unresolved?
122
+ raise "Some dependencies were not resolved: #{@dependency_graph.inspect}"
123
+ end
122
124
  end
123
125
  end
124
126
 
@@ -1,6 +1,18 @@
1
1
  module Dynflow
2
2
  module Executors
3
3
  class Abstract
4
+ Event = Algebrick.type do
5
+ fields! execution_plan_id: String,
6
+ step_id: Fixnum,
7
+ event: Object,
8
+ result: Future
9
+ end
10
+
11
+ Execution = Algebrick.type do
12
+ fields! execution_plan_id: String,
13
+ finished: Future
14
+ end
15
+
4
16
  include Algebrick::TypeCheck
5
17
  attr_reader :world, :logger
6
18
 
@@ -14,50 +14,31 @@ module Dynflow
14
14
 
15
15
  UnprocessableEvent = Class.new(Dynflow::Error)
16
16
 
17
- # actor messages
18
- Algebrick.types do
19
- Boolean = type { variants TrueClass, FalseClass }
17
+ Algebrick.type do |work|
18
+ Work = work
20
19
 
21
- Execution = type do
22
- fields! execution_plan_id: String,
23
- finished: Future
20
+ Work::Finalize = type do
21
+ fields! sequential_manager: SequentialManager,
22
+ execution_plan_id: String
24
23
  end
25
24
 
26
- Event = type do
27
- fields! execution_plan_id: String,
28
- step_id: Fixnum,
29
- event: Object,
30
- result: Future
25
+ Work::Step = type do
26
+ fields! step: ExecutionPlan::Steps::AbstractFlowStep,
27
+ execution_plan_id: String
31
28
  end
32
29
 
33
- Work = type do |work|
34
- work::Finalize = type do
35
- fields! sequential_manager: SequentialManager,
36
- execution_plan_id: String
37
- end
38
-
39
- work::Step = type do
40
- fields! step: ExecutionPlan::Steps::AbstractFlowStep,
41
- execution_plan_id: String
42
- end
43
-
44
- work::Event = type do
45
- fields! step: ExecutionPlan::Steps::AbstractFlowStep,
46
- execution_plan_id: String,
47
- event: Event
48
- end
49
-
50
- variants work::Step, work::Event, work::Finalize
30
+ Work::Event = type do
31
+ fields! step: ExecutionPlan::Steps::AbstractFlowStep,
32
+ execution_plan_id: String,
33
+ event: Event
51
34
  end
52
35
 
53
- PoolDone = type do
54
- fields! work: Work
55
- end
56
- WorkerDone = type do
57
- fields! work: Work, worker: Worker
58
- end
36
+ variants Work::Step, Work::Event, Work::Finalize
59
37
  end
60
38
 
39
+ PoolDone = Algebrick.type { fields! work: Work }
40
+ WorkerDone = Algebrick.type { fields! work: Work, worker: Worker }
41
+
61
42
  def initialize(world, pool_size = 10)
62
43
  super(world)
63
44
  @core = Core.new world, pool_size
@@ -18,11 +18,11 @@ module Dynflow
18
18
 
19
19
  def on_message(message)
20
20
  match message,
21
- (on ~Execution do |(execution_plan_id, finished)|
21
+ (on ~Parallel::Execution do |(execution_plan_id, finished)|
22
22
  start_executing track_execution_plan(execution_plan_id, finished)
23
23
  true
24
24
  end),
25
- (on ~Event do |event|
25
+ (on ~Parallel::Event do |event|
26
26
  event(event)
27
27
  end),
28
28
  (on PoolDone.(~any) do |step|
@@ -40,15 +40,18 @@ module Dynflow
40
40
  execution_plan = @world.persistence.load_execution_plan(execution_plan_id)
41
41
 
42
42
  if terminating?
43
- raise Dynflow::Error, "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
43
+ raise Dynflow::Error,
44
+ "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
44
45
  end
45
46
 
46
47
  if @execution_plan_managers[execution_plan_id]
47
- raise Dynflow::Error, "cannot execute execution_plan_id:#{execution_plan_id} it's already running"
48
+ raise Dynflow::Error,
49
+ "cannot execute execution_plan_id:#{execution_plan_id} it's already running"
48
50
  end
49
51
 
50
52
  if execution_plan.state == :stopped
51
- raise Dynflow::Error, "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
53
+ raise Dynflow::Error,
54
+ "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
52
55
  end
53
56
 
54
57
  @execution_plan_managers[execution_plan_id] =
@@ -95,14 +98,16 @@ module Dynflow
95
98
  end
96
99
 
97
100
  def event(event)
98
- Type! event, Event
101
+ Type! event, Parallel::Event
99
102
  execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
100
103
  if execution_plan_manager
101
104
  feed_pool execution_plan_manager.event(event)
102
105
  true
103
106
  else
104
- logger.warn "dropping event #{event} - no manager for #{event.execution_plan_id}:#{event.step_id}"
105
- event.result.fail UnprocessableEvent.new("no manager for #{event.execution_plan_id}:#{event.step_id}")
107
+ logger.warn format('dropping event %s - no manager for %s:%s',
108
+ event, event.execution_plan_id, event.step_id)
109
+ event.result.fail UnprocessableEvent.new(
110
+ "no manager for #{event.execution_plan_id}:#{event.step_id}")
106
111
  end
107
112
  end
108
113
 
@@ -46,38 +46,30 @@ module Dynflow
46
46
  end
47
47
  end
48
48
 
49
- match work,
50
-
51
- Work::Step.(step: ~any) >-> step do
52
- suspended, work = @running_steps_manager.done(step)
53
- if suspended
54
- raise 'assert' unless compute_next_from_step.call(step).empty?
55
- work
56
- else
57
- execution_plan.update_execution_time step.execution_time
58
- compute_next_from_step.call step
59
- end
60
- end,
61
-
62
- Work::Event.(step: ~any) >-> step do
63
- suspended, work = @running_steps_manager.done(step)
64
-
65
- if suspended
66
- work
67
- else
68
- execution_plan.update_execution_time step.execution_time
69
- compute_next_from_step.call step
70
- end
71
- end,
72
-
73
- Work::Finalize >-> do
74
- raise unless @finalize_manager
75
- finish
76
- end
49
+ match(work,
50
+ (on Work::Step.(step: ~any) | Work::Event.(step: ~any) do |step|
51
+ execution_plan.steps[step.id] = step
52
+ suspended, work = @running_steps_manager.done(step)
53
+ unless suspended
54
+ execution_plan.update_execution_time step.execution_time
55
+ work = compute_next_from_step.call step
56
+ end
57
+ # TODO: can be probably disabled to improve
58
+ # performance, execution time will not be updated,
59
+ # maybe more - check on the other side, it allows
60
+ # us to use persistence adapter for hooking into
61
+ # the running process.
62
+ execution_plan.save
63
+ work
64
+ end),
65
+ (on Work::Finalize do
66
+ raise unless @finalize_manager
67
+ finish
68
+ end))
77
69
  end
78
70
 
79
71
  def event(event)
80
- Type! event, Event
72
+ Type! event, Parallel::Event
81
73
  raise unless event.execution_plan_id == @execution_plan.id
82
74
  @running_steps_manager.event(event)
83
75
  end
@@ -19,9 +19,6 @@ module Dynflow
19
19
 
20
20
  # @return [Set] of steps to continue with
21
21
  def what_is_next(flow_step)
22
- execution_plan.steps[flow_step.id] = flow_step
23
- # TODO can be probably disabled to improve performance, execution time will not be updated, maybe more - check
24
- execution_plan.save
25
22
  return [] if flow_step.state == :suspended
26
23
 
27
24
  success = flow_step.state != :error
@@ -34,7 +34,8 @@ module Dynflow
34
34
  while (event = @events.shift(step.id))
35
35
  message = "step #{step.execution_plan_id}:#{step.id} dropping event #{event.event}"
36
36
  @world.logger.warn message
37
- event.event.result.fail UnprocessableEvent.new(message).tap { |e| e.set_backtrace(caller) }
37
+ event.event.result.fail UnprocessableEvent.new(message).
38
+ tap { |e| e.set_backtrace(caller) }
38
39
  end
39
40
  raise 'assert' unless @events.empty?(step.id)
40
41
  @running_steps.delete(step.id)
@@ -44,11 +45,12 @@ module Dynflow
44
45
 
45
46
  # @returns [Work, nil]
46
47
  def event(event)
47
- Type! event, Event
48
+ Type! event, Parallel::Event
48
49
 
49
50
  step = @running_steps[event.step_id]
50
51
  unless step
51
- event.result.fail UnprocessableEvent.new('step is not suspended, it cannot process events')
52
+ event.result.fail UnprocessableEvent.new(
53
+ 'step is not suspended, it cannot process events')
52
54
  return nil
53
55
  end
54
56
 
@@ -22,8 +22,12 @@ module Dynflow
22
22
  reset_finalize_steps
23
23
  unless execution_plan.error?
24
24
  world.transaction_adapter.transaction do
25
- unless dispatch(execution_plan.finalize_flow)
26
- world.transaction_adapter.rollback
25
+ step_id = execution_plan.finalize_flow.all_step_ids.first
26
+ action_class = execution_plan.steps[step_id].action_class
27
+ world.middleware.execute(:finalize_phase, action_class) do
28
+ unless dispatch(execution_plan.finalize_flow)
29
+ world.transaction_adapter.rollback
30
+ end
27
31
  end
28
32
  end
29
33
  end
@@ -15,12 +15,13 @@ module Dynflow
15
15
 
16
16
  def on_message(message)
17
17
  match message,
18
- Work::Step.(step: ~any) | Work::Event.(step: ~any, event: Event.(event: ~any)) >-> step, event do
18
+ (on Work::Step.(step: ~any) |
19
+ Work::Event.(step: ~any, event: Parallel::Event.(event: ~any)) do |step, event|
19
20
  step.execute event
20
- end,
21
- Work::Finalize.(~any, any) >-> sequential_manager do
21
+ end),
22
+ (on Work::Finalize.(~any, any) do |sequential_manager|
22
23
  sequential_manager.finalize
23
- end
24
+ end)
24
25
  @pool << WorkerDone[work: message, worker: self]
25
26
  @transaction_adapter.cleanup
26
27
  end
@@ -15,7 +15,7 @@ module Dynflow
15
15
  end
16
16
 
17
17
  def execute(execution_plan_id, finished = Future.new)
18
- @core.ask(Core::Execute[execution_plan_id, finished]).value!.value!
18
+ @core.ask(Core::Execution[execution_plan_id, finished]).value!.value!
19
19
  finished
20
20
  rescue => e
21
21
  finished.fail e unless finished.ready?
@@ -23,7 +23,8 @@ module Dynflow
23
23
  end
24
24
 
25
25
  def event(execution_plan_id, step_id, event, future = Future)
26
- raise 'events are handled in a process with real executor'
26
+ @core.ask(Core::Event[execution_plan_id, step_id, event, future]).value!
27
+ future
27
28
  end
28
29
 
29
30
  def terminate(future = Future.new)
@@ -33,6 +34,10 @@ module Dynflow
33
34
  def initialized
34
35
  @core.initialized
35
36
  end
37
+
38
+ def connected?
39
+ @core.ask(Core::Connect).value!
40
+ end
36
41
  end
37
42
  end
38
43
  end
@@ -5,16 +5,22 @@ module Dynflow
5
5
  include Listeners::Serialization
6
6
 
7
7
  Message = Algebrick.type do
8
+ Job = Algebrick.type do
9
+ variants Event = Executors::Abstract::Event,
10
+ Execution = Executors::Abstract::Execution
11
+ end
12
+
8
13
  variants Closed = atom,
9
- Received = type { fields message: SocketMessage },
10
- Execute = type { fields execution_plan_uuid: String, future: Future }
14
+ Received = type { fields message: Protocol::Response },
15
+ Connect = atom,
16
+ Job
11
17
  end
12
18
 
13
- Execution = Algebrick.type do
14
- fields! id: Integer, accepted: Future, finished: Future
19
+ TrackedJob = Algebrick.type do
20
+ fields! id: Integer, job: Protocol::Job, accepted: Future, finished: Future
15
21
  end
16
22
 
17
- module Execution
23
+ module TrackedJob
18
24
  def accept!
19
25
  accepted.resolve true
20
26
  self
@@ -26,9 +32,16 @@ module Dynflow
26
32
  self
27
33
  end
28
34
 
29
- def success!(value)
35
+ def success!(world)
30
36
  raise unless accepted.ready?
31
- finished.resolve value
37
+ finished.resolve(
38
+ match job,
39
+ (on Core::Protocol::Execution.(execution_plan_id: ~any) do |uuid|
40
+ world.persistence.load_execution_plan(uuid)
41
+ end),
42
+ (on Core::Protocol::Event do
43
+ true
44
+ end))
32
45
  self
33
46
  end
34
47
 
@@ -49,26 +62,34 @@ module Dynflow
49
62
  private
50
63
 
51
64
  def delayed_initialize(world, socket_path)
52
- @socket_path = Type! socket_path, String
53
- @world = Type! world, World
54
- @socket = nil
55
- @last_id = 0
56
- @executions = {}
65
+ @socket_path = Type! socket_path, String
66
+ @world = Type! world, World
67
+ @socket = nil
68
+ @last_id = 0
69
+ @tracked_jobs = {}
57
70
  connect
58
71
  end
59
72
 
60
73
  def termination
61
- disconnect
74
+ terminate! if disconnect
62
75
  end
63
76
 
64
77
  def on_message(message)
78
+ Type! message, Message
65
79
  match message,
66
-
67
- (on Core::Execute.(~any, ~any) do |execution_plan_uuid, future|
80
+ (on ~Job do |job|
68
81
  raise 'terminating' if terminating?
69
- id, accepted = add_execution future
82
+ job, future =
83
+ match job,
84
+ (on ~Execution do |(execution_plan_uuid, future)|
85
+ [Protocol::Execution[execution_plan_uuid], future]
86
+ end),
87
+ (on ~Event do |(execution_plan_id, step_id, event, future)|
88
+ [Protocol::Event[execution_plan_id, step_id, event], future]
89
+ end)
90
+ id, accepted = add_tracked_job future, job
70
91
  success = connect && begin
71
- send_message @socket, RemoteViaSocket::Execute[id, execution_plan_uuid]
92
+ send_message @socket, Protocol::Do[id, job]
72
93
  true
73
94
  rescue IOError => error
74
95
  logger.warn error
@@ -76,36 +97,43 @@ module Dynflow
76
97
  end
77
98
 
78
99
  unless success
79
- @executions[id].reject! Dynflow::Error.new(
80
- 'No connection to RemoteViaSocket::Listener')
100
+ @tracked_jobs[id].reject!(
101
+ Dynflow::Error.new(
102
+ "Cannot do #{message}, no connection to a Listener"))
81
103
  end
82
104
 
83
105
  return accepted
84
106
  end),
85
107
 
86
- (on Received.(Accepted.(~any)) do |id|
87
- @executions[id].accept!
108
+ (on Received.(~Protocol::Accepted) do |(id)|
109
+ @tracked_jobs[id].accept!
88
110
  end),
89
111
 
90
- (on Received.(Failed.(~any, ~any)) do |id, error|
91
- @executions.delete(id).reject! Dynflow::Error.new(error)
112
+ (on Received.(~Protocol::Failed) do |(id, error)|
113
+ @tracked_jobs.delete(id).reject! Dynflow::Error.new(error)
92
114
  end),
93
115
 
94
- (on Received.(Done.(~any, ~any)) do |id, uuid|
95
- @executions.delete(id).success! @world.persistence.load_execution_plan(uuid)
116
+ (on Received.(~Protocol::Done) do |(id)|
117
+ @tracked_jobs.delete(id).success! @world
96
118
  end),
97
119
 
98
120
  (on Closed do
99
121
  @socket = nil
100
122
  logger.info 'Disconnected from server.'
101
- @executions.each { |_, c| c.fail! 'No connection to RemoteViaSocket::Listener' }
102
- @executions.clear
123
+ @tracked_jobs.each do |_, c|
124
+ c.fail! 'Connection to a Listener lost.'
125
+ end
126
+ @tracked_jobs.clear
103
127
  terminate! if terminating?
128
+ end),
129
+
130
+ (on Connect do
131
+ connect
104
132
  end)
105
133
  end
106
134
 
107
- def add_execution(finished)
108
- @executions[id = (@last_id += 1)] = Execution[id, accepted = Future.new, finished]
135
+ def add_tracked_job(finished, job)
136
+ @tracked_jobs[id = (@last_id += 1)] = TrackedJob[id, job, accepted = Future.new, finished]
109
137
  return id, accepted
110
138
  end
111
139
 
@@ -115,14 +143,20 @@ module Dynflow
115
143
  logger.info 'Connected to server.'
116
144
  read_socket_until_closed
117
145
  true
118
- rescue IOError => error
146
+ rescue SystemCallError, IOError => error
119
147
  logger.warn error
120
148
  false
149
+ rescue => error
150
+ logger.fatal error
151
+ raise error
121
152
  end
122
153
 
123
154
  def disconnect
124
155
  return true unless @socket
125
- @socket.shutdown :RDWR
156
+
157
+ @socket.close
158
+ false
159
+ rescue Errno::ENOTCONN
126
160
  true
127
161
  end
128
162
 
@@ -136,7 +170,7 @@ module Dynflow
136
170
 
137
171
  def read_socket
138
172
  match message = receive_message(@socket),
139
- SocketMessage >-> { self << Received[message] },
173
+ Protocol::Message >-> { self << Received[message] },
140
174
  NilClass.to_m >-> do
141
175
  self << Closed
142
176
  throw :stop_reading