dynflow 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +1 -1
  3. data/.travis.yml +3 -4
  4. data/Dockerfile +9 -0
  5. data/Gemfile +6 -0
  6. data/Rakefile +1 -0
  7. data/doc/pages/Gemfile +1 -0
  8. data/doc/pages/Rakefile +1 -0
  9. data/doc/pages/plugins/alert_block.rb +1 -0
  10. data/doc/pages/plugins/div_tag.rb +1 -0
  11. data/doc/pages/plugins/graphviz.rb +1 -0
  12. data/doc/pages/plugins/plantuml.rb +1 -0
  13. data/doc/pages/plugins/play.rb +1 -0
  14. data/doc/pages/plugins/tags.rb +1 -0
  15. data/doc/pages/plugins/toc.rb +1 -0
  16. data/docker-compose.yml +41 -0
  17. data/dynflow.gemspec +1 -0
  18. data/examples/clock_benchmark.rb +1 -0
  19. data/examples/example_helper.rb +19 -2
  20. data/examples/future_execution.rb +2 -1
  21. data/examples/memory_limit_watcher.rb +1 -0
  22. data/examples/orchestrate.rb +4 -5
  23. data/examples/orchestrate_evented.rb +3 -2
  24. data/examples/remote_executor.rb +68 -0
  25. data/examples/singletons.rb +4 -3
  26. data/examples/sub_plan_concurrency_control.rb +2 -1
  27. data/examples/sub_plans.rb +3 -2
  28. data/examples/termination.rb +1 -0
  29. data/lib/dynflow.rb +20 -0
  30. data/lib/dynflow/action.rb +28 -3
  31. data/lib/dynflow/action/cancellable.rb +1 -0
  32. data/lib/dynflow/action/format.rb +1 -0
  33. data/lib/dynflow/action/missing.rb +1 -0
  34. data/lib/dynflow/action/polling.rb +3 -1
  35. data/lib/dynflow/action/progress.rb +1 -0
  36. data/lib/dynflow/action/rescue.rb +1 -0
  37. data/lib/dynflow/action/singleton.rb +1 -0
  38. data/lib/dynflow/action/suspended.rb +9 -2
  39. data/lib/dynflow/action/timeouts.rb +2 -1
  40. data/lib/dynflow/action/with_bulk_sub_plans.rb +2 -1
  41. data/lib/dynflow/action/with_polling_sub_plans.rb +7 -5
  42. data/lib/dynflow/action/with_sub_plans.rb +1 -0
  43. data/lib/dynflow/active_job/queue_adapter.rb +1 -0
  44. data/lib/dynflow/actor.rb +13 -5
  45. data/lib/dynflow/actors.rb +1 -0
  46. data/lib/dynflow/actors/execution_plan_cleaner.rb +1 -0
  47. data/lib/dynflow/clock.rb +27 -47
  48. data/lib/dynflow/config.rb +11 -2
  49. data/lib/dynflow/connectors.rb +1 -0
  50. data/lib/dynflow/connectors/abstract.rb +1 -0
  51. data/lib/dynflow/connectors/database.rb +1 -0
  52. data/lib/dynflow/connectors/direct.rb +1 -0
  53. data/lib/dynflow/coordinator.rb +1 -0
  54. data/lib/dynflow/coordinator_adapters.rb +1 -0
  55. data/lib/dynflow/coordinator_adapters/abstract.rb +1 -0
  56. data/lib/dynflow/coordinator_adapters/sequel.rb +1 -0
  57. data/lib/dynflow/dead_letter_silencer.rb +2 -0
  58. data/lib/dynflow/debug/telemetry/persistence.rb +1 -0
  59. data/lib/dynflow/delayed_executors.rb +1 -0
  60. data/lib/dynflow/delayed_executors/abstract.rb +1 -0
  61. data/lib/dynflow/delayed_executors/abstract_core.rb +1 -0
  62. data/lib/dynflow/delayed_executors/polling.rb +1 -0
  63. data/lib/dynflow/delayed_plan.rb +1 -0
  64. data/lib/dynflow/director.rb +80 -15
  65. data/lib/dynflow/director/execution_plan_manager.rb +17 -3
  66. data/lib/dynflow/director/flow_manager.rb +1 -0
  67. data/lib/dynflow/director/{work_queue.rb → queue_hash.rb} +9 -8
  68. data/lib/dynflow/director/running_steps_manager.rb +55 -18
  69. data/lib/dynflow/director/sequence_cursor.rb +1 -0
  70. data/lib/dynflow/director/sequential_manager.rb +12 -2
  71. data/lib/dynflow/dispatcher.rb +4 -2
  72. data/lib/dynflow/dispatcher/abstract.rb +1 -0
  73. data/lib/dynflow/dispatcher/client_dispatcher.rb +6 -4
  74. data/lib/dynflow/dispatcher/executor_dispatcher.rb +13 -1
  75. data/lib/dynflow/errors.rb +1 -0
  76. data/lib/dynflow/execution_history.rb +1 -0
  77. data/lib/dynflow/execution_plan.rb +3 -2
  78. data/lib/dynflow/execution_plan/dependency_graph.rb +1 -0
  79. data/lib/dynflow/execution_plan/hooks.rb +1 -0
  80. data/lib/dynflow/execution_plan/output_reference.rb +2 -1
  81. data/lib/dynflow/execution_plan/steps.rb +1 -0
  82. data/lib/dynflow/execution_plan/steps/abstract.rb +10 -5
  83. data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb +2 -0
  84. data/lib/dynflow/execution_plan/steps/error.rb +1 -0
  85. data/lib/dynflow/execution_plan/steps/finalize_step.rb +1 -0
  86. data/lib/dynflow/execution_plan/steps/plan_step.rb +1 -0
  87. data/lib/dynflow/execution_plan/steps/run_step.rb +1 -0
  88. data/lib/dynflow/executors.rb +1 -1
  89. data/lib/dynflow/executors/abstract/core.rb +132 -0
  90. data/lib/dynflow/executors/parallel.rb +24 -11
  91. data/lib/dynflow/executors/parallel/core.rb +10 -91
  92. data/lib/dynflow/executors/parallel/pool.rb +4 -2
  93. data/lib/dynflow/executors/parallel/worker.rb +2 -1
  94. data/lib/dynflow/executors/sidekiq/core.rb +121 -0
  95. data/lib/dynflow/executors/sidekiq/internal_job_base.rb +24 -0
  96. data/lib/dynflow/executors/sidekiq/orchestrator_jobs.rb +60 -0
  97. data/lib/dynflow/executors/sidekiq/redis_locking.rb +69 -0
  98. data/lib/dynflow/executors/sidekiq/serialization.rb +33 -0
  99. data/lib/dynflow/executors/sidekiq/worker_jobs.rb +42 -0
  100. data/lib/dynflow/flows.rb +1 -0
  101. data/lib/dynflow/flows/abstract.rb +1 -0
  102. data/lib/dynflow/flows/abstract_composed.rb +1 -0
  103. data/lib/dynflow/flows/atom.rb +1 -0
  104. data/lib/dynflow/flows/concurrence.rb +1 -0
  105. data/lib/dynflow/flows/sequence.rb +1 -0
  106. data/lib/dynflow/logger_adapters.rb +1 -0
  107. data/lib/dynflow/logger_adapters/abstract.rb +1 -0
  108. data/lib/dynflow/logger_adapters/delegator.rb +1 -0
  109. data/lib/dynflow/logger_adapters/formatters.rb +1 -0
  110. data/lib/dynflow/logger_adapters/formatters/abstract.rb +1 -0
  111. data/lib/dynflow/logger_adapters/formatters/exception.rb +1 -0
  112. data/lib/dynflow/logger_adapters/simple.rb +1 -0
  113. data/lib/dynflow/middleware.rb +1 -0
  114. data/lib/dynflow/middleware/common/singleton.rb +1 -0
  115. data/lib/dynflow/middleware/common/transaction.rb +1 -0
  116. data/lib/dynflow/middleware/register.rb +1 -0
  117. data/lib/dynflow/middleware/resolver.rb +1 -0
  118. data/lib/dynflow/middleware/stack.rb +1 -0
  119. data/lib/dynflow/middleware/world.rb +1 -0
  120. data/lib/dynflow/persistence.rb +3 -2
  121. data/lib/dynflow/persistence_adapters.rb +1 -0
  122. data/lib/dynflow/persistence_adapters/abstract.rb +1 -0
  123. data/lib/dynflow/persistence_adapters/sequel.rb +10 -7
  124. data/lib/dynflow/persistence_adapters/sequel_migrations/001_initial.rb +1 -0
  125. data/lib/dynflow/persistence_adapters/sequel_migrations/002_incremental_progress.rb +1 -0
  126. data/lib/dynflow/persistence_adapters/sequel_migrations/003_parent_action.rb +1 -0
  127. data/lib/dynflow/persistence_adapters/sequel_migrations/004_coordinator_records.rb +1 -0
  128. data/lib/dynflow/persistence_adapters/sequel_migrations/005_envelopes.rb +1 -0
  129. data/lib/dynflow/persistence_adapters/sequel_migrations/006_fix_data_length.rb +1 -0
  130. data/lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb +1 -0
  131. data/lib/dynflow/persistence_adapters/sequel_migrations/008_rename_scheduled_plans_to_delayed_plans.rb +1 -0
  132. data/lib/dynflow/persistence_adapters/sequel_migrations/009_fix_mysql_data_length.rb +1 -0
  133. data/lib/dynflow/persistence_adapters/sequel_migrations/010_add_execution_plans_label.rb +1 -0
  134. data/lib/dynflow/persistence_adapters/sequel_migrations/011_placeholder.rb +1 -0
  135. data/lib/dynflow/persistence_adapters/sequel_migrations/012_add_delayed_plans_serialized_args.rb +1 -0
  136. data/lib/dynflow/persistence_adapters/sequel_migrations/013_add_action_columns.rb +1 -0
  137. data/lib/dynflow/persistence_adapters/sequel_migrations/014_add_step_columns.rb +1 -0
  138. data/lib/dynflow/persistence_adapters/sequel_migrations/015_add_execution_plan_columns.rb +1 -0
  139. data/lib/dynflow/persistence_adapters/sequel_migrations/016_add_step_queue.rb +1 -0
  140. data/lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb +1 -0
  141. data/lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb +1 -0
  142. data/lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb +48 -0
  143. data/lib/dynflow/rails.rb +1 -0
  144. data/lib/dynflow/rails/configuration.rb +6 -3
  145. data/lib/dynflow/rails/daemon.rb +1 -0
  146. data/lib/dynflow/round_robin.rb +1 -0
  147. data/lib/dynflow/semaphores.rb +1 -0
  148. data/lib/dynflow/semaphores/abstract.rb +1 -0
  149. data/lib/dynflow/semaphores/aggregating.rb +1 -0
  150. data/lib/dynflow/semaphores/dummy.rb +1 -0
  151. data/lib/dynflow/semaphores/stateful.rb +1 -0
  152. data/lib/dynflow/serializable.rb +13 -4
  153. data/lib/dynflow/serializer.rb +24 -0
  154. data/lib/dynflow/serializers.rb +1 -0
  155. data/lib/dynflow/serializers/abstract.rb +1 -0
  156. data/lib/dynflow/serializers/noop.rb +1 -0
  157. data/lib/dynflow/stateful.rb +1 -0
  158. data/lib/dynflow/telemetry.rb +1 -0
  159. data/lib/dynflow/telemetry_adapters/abstract.rb +1 -0
  160. data/lib/dynflow/telemetry_adapters/dummy.rb +1 -0
  161. data/lib/dynflow/telemetry_adapters/statsd.rb +1 -0
  162. data/lib/dynflow/testing.rb +1 -0
  163. data/lib/dynflow/testing/assertions.rb +6 -5
  164. data/lib/dynflow/testing/dummy_execution_plan.rb +1 -0
  165. data/lib/dynflow/testing/dummy_executor.rb +19 -2
  166. data/lib/dynflow/testing/dummy_planned_action.rb +1 -0
  167. data/lib/dynflow/testing/dummy_step.rb +3 -1
  168. data/lib/dynflow/testing/dummy_world.rb +9 -0
  169. data/lib/dynflow/testing/factories.rb +6 -1
  170. data/lib/dynflow/testing/in_thread_executor.rb +22 -3
  171. data/lib/dynflow/testing/in_thread_world.rb +9 -0
  172. data/lib/dynflow/testing/managed_clock.rb +1 -0
  173. data/lib/dynflow/testing/mimic.rb +1 -0
  174. data/lib/dynflow/throttle_limiter.rb +1 -0
  175. data/lib/dynflow/transaction_adapters.rb +1 -0
  176. data/lib/dynflow/transaction_adapters/abstract.rb +1 -0
  177. data/lib/dynflow/transaction_adapters/active_record.rb +1 -0
  178. data/lib/dynflow/transaction_adapters/none.rb +1 -0
  179. data/lib/dynflow/utils.rb +1 -0
  180. data/lib/dynflow/utils/indifferent_hash.rb +1 -0
  181. data/lib/dynflow/utils/priority_queue.rb +1 -0
  182. data/lib/dynflow/version.rb +2 -1
  183. data/lib/dynflow/watchers/memory_consumption_watcher.rb +1 -0
  184. data/lib/dynflow/web.rb +1 -0
  185. data/lib/dynflow/web/console.rb +1 -0
  186. data/lib/dynflow/web/console_helpers.rb +1 -0
  187. data/lib/dynflow/web/filtering_helpers.rb +1 -0
  188. data/lib/dynflow/web/world_helpers.rb +1 -0
  189. data/lib/dynflow/web_console.rb +1 -0
  190. data/lib/dynflow/world.rb +11 -1
  191. data/lib/dynflow/world/invalidation.rb +7 -1
  192. data/test/abnormal_states_recovery_test.rb +41 -40
  193. data/test/action_test.rb +160 -110
  194. data/test/activejob_adapter_test.rb +1 -0
  195. data/test/batch_sub_tasks_test.rb +12 -11
  196. data/test/clock_test.rb +2 -1
  197. data/test/concurrency_control_test.rb +20 -19
  198. data/test/coordinator_test.rb +20 -21
  199. data/test/daemon_test.rb +2 -1
  200. data/test/dead_letter_silencer_test.rb +9 -7
  201. data/test/dispatcher_test.rb +2 -1
  202. data/test/execution_plan_cleaner_test.rb +13 -12
  203. data/test/execution_plan_hooks_test.rb +3 -2
  204. data/test/execution_plan_test.rb +33 -32
  205. data/test/executor_test.rb +533 -489
  206. data/test/future_execution_test.rb +45 -44
  207. data/test/memory_cosumption_watcher_test.rb +5 -4
  208. data/test/middleware_test.rb +55 -54
  209. data/test/persistence_test.rb +56 -53
  210. data/test/rescue_test.rb +36 -35
  211. data/test/round_robin_test.rb +13 -12
  212. data/test/semaphores_test.rb +31 -30
  213. data/test/support/code_workflow_example.rb +1 -0
  214. data/test/support/dummy_example.rb +14 -1
  215. data/test/support/middleware_example.rb +2 -1
  216. data/test/support/rails/config/environment.rb +1 -0
  217. data/test/support/rescue_example.rb +1 -0
  218. data/test/support/test_execution_log.rb +1 -0
  219. data/test/test_helper.rb +18 -17
  220. data/test/testing_test.rb +45 -44
  221. data/test/utils_test.rb +18 -17
  222. data/test/web_console_test.rb +1 -0
  223. data/test/world_test.rb +7 -6
  224. metadata +13 -4
  225. data/lib/dynflow/executors/abstract.rb +0 -40
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  class Director
3
4
  class SequenceCursor
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  class Director
3
4
  class SequentialManager
@@ -30,13 +31,22 @@ module Dynflow
30
31
  @done = true
31
32
  end
32
33
 
34
+ def finalize_steps
35
+ execution_plan.finalize_flow.all_step_ids.map do |step_id|
36
+ execution_plan.steps[step_id]
37
+ end
38
+ end
39
+
33
40
  def reset_finalize_steps
34
- execution_plan.finalize_flow.all_step_ids.each do |step_id|
35
- step = execution_plan.steps[step_id]
41
+ finalize_steps.each do |step|
36
42
  step.state = :pending if [:success, :error].include? step.state
37
43
  end
38
44
  end
39
45
 
46
+ def done!
47
+ @done = true
48
+ end
49
+
40
50
  def done?
41
51
  @done
42
52
  end
@@ -1,10 +1,12 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module Dispatcher
3
4
  Request = Algebrick.type do
4
5
  Event = type do
5
6
  fields! execution_plan_id: String,
6
7
  step_id: Integer,
7
- event: Object
8
+ event: Object,
9
+ time: type { variants Time, NilClass }
8
10
  end
9
11
 
10
12
  Execution = type do
@@ -33,7 +35,7 @@ module Dynflow
33
35
  end
34
36
 
35
37
  Envelope = Algebrick.type do
36
- fields! request_id: Integer,
38
+ fields! request_id: String,
37
39
  sender_id: String,
38
40
  receiver_id: type { variants String, AnyExecutor = atom, UnknownWorld = atom },
39
41
  message: type { variants Request, Response }
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module Dispatcher
3
4
  class Abstract < Actor
@@ -1,9 +1,10 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module Dispatcher
3
4
  class ClientDispatcher < Abstract
4
5
 
5
6
  TrackedRequest = Algebrick.type do
6
- fields! id: Integer, request: Request,
7
+ fields! id: String, request: Request,
7
8
  accepted: Concurrent::Promises::ResolvableFuture, finished: Concurrent::Promises::ResolvableFuture
8
9
  end
9
10
 
@@ -33,7 +34,7 @@ module Dynflow
33
34
  # or whether the Ping really needs to be sent.
34
35
  class PingCache
35
36
  # Format string used for formating and parsing times
36
- TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%L'.freeze
37
+ TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%L'
37
38
  DEFAULT_MAX_AGE = 60
38
39
 
39
40
  # Formats time into a string
@@ -105,7 +106,7 @@ module Dynflow
105
106
  attr_reader :ping_cache
106
107
  def initialize(world, ping_cache_age)
107
108
  @world = Type! world, World
108
- @last_id = 0
109
+ @last_id_suffix = 0
109
110
  @tracked_requests = {}
110
111
  @terminated = nil
111
112
  @ping_cache = PingCache.new world, ping_cache_age
@@ -197,7 +198,8 @@ module Dynflow
197
198
  end
198
199
 
199
200
  def track_request(finished, request, timeout)
200
- id = @last_id += 1
201
+ id_suffix = @last_id_suffix += 1
202
+ id = "#{@world.id}-#{id_suffix}"
201
203
  tracked_request = TrackedRequest[id, request, Concurrent::Promises.resolvable_future, finished]
202
204
  @tracked_requests[id] = tracked_request
203
205
  @world.clock.ping(self, timeout, [:timeout, id]) if timeout
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module Dispatcher
3
4
  class ExecutorDispatcher < Abstract
@@ -50,7 +51,18 @@ module Dynflow
50
51
  respond(envelope, Failed[reason.to_s])
51
52
  end
52
53
  end
53
- @world.executor.event(event_request.execution_plan_id, event_request.step_id, event_request.event, future)
54
+ if event_request.time.nil? || event_request.time < Time.now
55
+ @world.executor.event(envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, future)
56
+ else
57
+ @world.clock.ping(
58
+ @world.executor,
59
+ event_request.time,
60
+ Director::Event[envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, Concurrent::Promises.resolvable_future],
61
+ :delayed_event
62
+ )
63
+ # resolves the future right away - currently we do not wait for the clock ping
64
+ future.fulfill true
65
+ end
54
66
  rescue Dynflow::Error => e
55
67
  future.reject(e) if future && !future.resolved?
56
68
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module Errors
3
4
  class RescueError < StandardError; end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  class ExecutionHistory
3
4
  include Algebrick::TypeCheck
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'securerandom'
2
3
 
3
4
  module Dynflow
@@ -121,9 +122,9 @@ module Dynflow
121
122
  original = self.state
122
123
  case self.state = state
123
124
  when :planning
124
- @started_at = Time.now
125
+ @started_at = Time.now.utc
125
126
  when :stopped
126
- @ended_at = Time.now
127
+ @ended_at = Time.now.utc
127
128
  @real_time = @ended_at - @started_at unless @started_at.nil?
128
129
  @execution_time = compute_execution_time
129
130
  key = failure? ? :failure : :success
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  class ExecutionPlan::DependencyGraph
3
4
 
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  class ExecutionPlan
3
4
  module Hooks
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  class ExecutionPlan::OutputReference < Serializable
3
4
  include Algebrick::TypeCheck
@@ -59,7 +60,7 @@ module Dynflow
59
60
  end
60
61
 
61
62
  def to_s
62
- "Step(#{step_id}).output".tap do |ret|
63
+ "Step(#{step_id}).output".dup.tap do |ret|
63
64
  ret << subkeys.map { |k| "[:#{k}]" }.join('') if subkeys.any?
64
65
  end
65
66
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module ExecutionPlan::Steps
3
4
 
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module ExecutionPlan::Steps
3
4
  class Abstract < Serializable
@@ -5,7 +6,7 @@ module Dynflow
5
6
  include Stateful
6
7
 
7
8
  attr_reader :execution_plan_id, :id, :state, :action_class, :action_id, :world, :started_at,
8
- :ended_at, :execution_time, :real_time, :queue
9
+ :ended_at, :execution_time, :real_time, :queue, :delayed_events
9
10
  attr_accessor :error
10
11
 
11
12
  # rubocop:disable Metrics/ParameterLists
@@ -47,6 +48,10 @@ module Dynflow
47
48
  end
48
49
  # rubocop:enable Metrics/ParameterLists
49
50
 
51
+ def ==(other)
52
+ other.class == self.class && other.execution_plan_id == self.execution_plan_id && other.id == self.id
53
+ end
54
+
50
55
  def action_logger
51
56
  @world.action_logger
52
57
  end
@@ -63,8 +68,8 @@ module Dynflow
63
68
  world.persistence
64
69
  end
65
70
 
66
- def save
67
- persistence.save_step(self)
71
+ def save(conditions = {})
72
+ persistence.save_step(self, conditions)
68
73
  end
69
74
 
70
75
  def self.states
@@ -157,12 +162,12 @@ module Dynflow
157
162
  private
158
163
 
159
164
  def with_meta_calculation(action, &block)
160
- start = Time.now
165
+ start = Time.now.utc
161
166
  @started_at ||= start
162
167
  block.call
163
168
  ensure
164
169
  calculate_progress(action)
165
- @ended_at = Time.now
170
+ @ended_at = Time.now.utc
166
171
  current_execution_time = @ended_at - start
167
172
  @execution_time += current_execution_time
168
173
  @real_time = @ended_at - @started_at
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module ExecutionPlan::Steps
3
4
  class AbstractFlowStep < Abstract
@@ -15,6 +16,7 @@ module Dynflow
15
16
  open_action do |action|
16
17
  with_meta_calculation(action) do
17
18
  action.execute(*args)
19
+ @delayed_events = action.delayed_events
18
20
  end
19
21
  end
20
22
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module ExecutionPlan::Steps
3
4
  class Error < Serializable
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module ExecutionPlan::Steps
3
4
  class FinalizeStep < AbstractFlowStep
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module ExecutionPlan::Steps
3
4
  class PlanStep < Abstract
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module ExecutionPlan::Steps
3
4
  class RunStep < AbstractFlowStep
@@ -1,7 +1,7 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module Executors
3
4
 
4
- require 'dynflow/executors/abstract'
5
5
  require 'dynflow/executors/parallel'
6
6
 
7
7
  # Every time we run a code that can be defined outside of Dynflow,
@@ -0,0 +1,132 @@
1
+ # frozen_string_literal: true
2
+ module Dynflow
3
+ module Executors
4
+ module Abstract
5
+ class Core < Actor
6
+ attr_reader :logger
7
+
8
+ def initialize(world, heartbeat_interval, queues_options)
9
+ @logger = world.logger
10
+ @world = Type! world, World
11
+ @pools = {}
12
+ @terminated = nil
13
+ @director = Director.new(@world)
14
+ @heartbeat_interval = heartbeat_interval
15
+ @queues_options = queues_options
16
+
17
+ schedule_heartbeat
18
+ end
19
+
20
+ def handle_execution(execution_plan_id, finished)
21
+ if terminating?
22
+ raise Dynflow::Error,
23
+ "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
24
+ end
25
+
26
+ handle_work(@director.start_execution(execution_plan_id, finished))
27
+ end
28
+
29
+ def handle_event(event)
30
+ Type! event, Director::Event
31
+ if terminating?
32
+ raise Dynflow::Error,
33
+ "cannot accept event: #{event} core is terminating"
34
+ end
35
+ handle_work(@director.handle_event(event))
36
+ end
37
+
38
+ def plan_events(delayed_events)
39
+ delayed_events.each do |event|
40
+ @world.plan_event(event.execution_plan_id, event.step_id, event.event, event.time)
41
+ end
42
+ end
43
+
44
+ def work_finished(work, delayed_events = nil)
45
+ handle_work(@director.work_finished(work))
46
+ plan_events(delayed_events) if delayed_events
47
+ end
48
+
49
+ def handle_persistence_error(error, work = nil)
50
+ logger.error "PersistenceError in executor"
51
+ logger.error error
52
+ @director.work_failed(work) if work
53
+ if error.is_a? Errors::FatalPersistenceError
54
+ logger.fatal "Terminating"
55
+ @world.terminate
56
+ end
57
+ end
58
+
59
+ def start_termination(*args)
60
+ logger.info 'shutting down Core ...'
61
+ super
62
+ end
63
+
64
+ def finish_termination
65
+ @director.terminate
66
+ logger.info '... Dynflow core terminated.'
67
+ super()
68
+ end
69
+
70
+ def dead_letter_routing
71
+ @world.dead_letter_handler
72
+ end
73
+
74
+ def execution_status(execution_plan_id = nil)
75
+ {}
76
+ end
77
+
78
+ def heartbeat
79
+ @logger.debug('Executor heartbeat')
80
+ record = @world.coordinator.find_records(:id => @world.id,
81
+ :class => ['Dynflow::Coordinator::ExecutorWorld', 'Dynflow::Coordinator::ClientWorld']).first
82
+ unless record
83
+ logger.error(%{Executor's world record for #{@world.id} missing: terminating})
84
+ @world.terminate
85
+ return
86
+ end
87
+
88
+ record.data[:meta].update(:last_seen => Dynflow::Dispatcher::ClientDispatcher::PingCache.format_time)
89
+ @world.coordinator.update_record(record)
90
+ schedule_heartbeat
91
+ end
92
+
93
+ private
94
+
95
+ def suggest_queue(work_item)
96
+ queue = work_item.queue
97
+ unless @queues_options.key?(queue)
98
+ logger.debug("Pool is not available for queue #{queue}, falling back to #{fallback_queue}")
99
+ queue = fallback_queue
100
+ end
101
+ queue
102
+ end
103
+
104
+ def fallback_queue
105
+ :default
106
+ end
107
+
108
+ def schedule_heartbeat
109
+ @world.clock.ping(self, @heartbeat_interval, :heartbeat)
110
+ end
111
+
112
+ def on_message(message)
113
+ super
114
+ rescue Errors::PersistenceError => e
115
+ handle_persistence_error(e)
116
+ end
117
+
118
+ def handle_work(work_items)
119
+ return if terminating?
120
+ return if work_items.nil?
121
+ work_items = [work_items] if work_items.is_a? Director::WorkItem
122
+ work_items.all? { |i| Type! i, Director::WorkItem }
123
+ feed_pool(work_items)
124
+ end
125
+
126
+ def feed_pool(work_items)
127
+ raise NotImplementedError
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
@@ -1,15 +1,23 @@
1
+ # frozen_string_literal: true
1
2
  module Dynflow
2
3
  module Executors
3
- class Parallel < Abstract
4
+ class Parallel
5
+ require 'dynflow/executors/abstract/core'
4
6
  require 'dynflow/executors/parallel/core'
5
- require 'dynflow/executors/parallel/pool'
6
- require 'dynflow/executors/parallel/worker'
7
-
8
- def initialize(world, heartbeat_interval, queues_options = { :default => { :pool_size => 5 }})
9
- super(world)
10
- @core = Core.spawn name: 'parallel-executor-core',
11
- args: [world, heartbeat_interval, queues_options],
12
- initialized: @core_initialized = Concurrent::Promises.resolvable_future
7
+ # only load Sidekiq pieces when run in Sidekiq runtime (and the Sidekiq module is already loaded)
8
+ require 'dynflow/executors/sidekiq/core' if defined? ::Sidekiq
9
+
10
+ attr_reader :core
11
+
12
+ def initialize(world,
13
+ executor_class:,
14
+ heartbeat_interval:,
15
+ queues_options: { :default => { :pool_size => 5 }})
16
+ @world = world
17
+ @logger = world.logger
18
+ @core = executor_class.spawn name: 'parallel-executor-core',
19
+ args: [world, heartbeat_interval, queues_options],
20
+ initialized: @core_initialized = Concurrent::Promises.resolvable_future
13
21
  end
14
22
 
15
23
  def execute(execution_plan_id, finished = Concurrent::Promises.resolvable_future, wait_for_acceptance = true)
@@ -25,11 +33,16 @@ module Dynflow
25
33
  raise e
26
34
  end
27
35
 
28
- def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
29
- @core.ask([:handle_event, Director::Event[execution_plan_id, step_id, event, future]])
36
+ def event(request_id, execution_plan_id, step_id, event, future = nil)
37
+ @core.ask([:handle_event, Director::Event[request_id, execution_plan_id, step_id, event, future]])
30
38
  future
31
39
  end
32
40
 
41
+ def delayed_event(director_event)
42
+ @core.ask([:handle_event, director_event])
43
+ director_event.result
44
+ end
45
+
33
46
  def terminate(future = Concurrent::Promises.resolvable_future)
34
47
  @core.tell([:start_termination, future])
35
48
  future