dynflow 1.8.1 → 1.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +1 -1
- data/.rubocop.yml +11 -5
- data/.rubocop_todo.yml +777 -345
- data/Gemfile +4 -3
- data/Rakefile +1 -0
- data/doc/pages/Gemfile +4 -3
- data/doc/pages/Rakefile +1 -0
- data/doc/pages/plugins/alert_block.rb +1 -0
- data/doc/pages/plugins/div_tag.rb +1 -0
- data/doc/pages/plugins/graphviz.rb +6 -6
- data/doc/pages/plugins/plantuml.rb +2 -3
- data/doc/pages/plugins/play.rb +1 -2
- data/doc/pages/plugins/tags.rb +2 -8
- data/doc/pages/plugins/toc.rb +1 -1
- data/dynflow.gemspec +11 -10
- data/examples/clock_benchmark.rb +1 -0
- data/examples/example_helper.rb +4 -3
- data/examples/future_execution.rb +0 -2
- data/examples/memory_limit_watcher.rb +8 -8
- data/examples/orchestrate.rb +9 -21
- data/examples/orchestrate_evented.rb +18 -33
- data/examples/remote_executor.rb +13 -14
- data/examples/singletons.rb +1 -0
- data/examples/sub_plan_concurrency_control.rb +0 -1
- data/examples/sub_plans.rb +1 -0
- data/examples/sub_plans_v2.rb +1 -0
- data/lib/dynflow/action/cancellable.rb +1 -0
- data/lib/dynflow/action/format.rb +1 -4
- data/lib/dynflow/action/missing.rb +4 -4
- data/lib/dynflow/action/polling.rb +2 -3
- data/lib/dynflow/action/progress.rb +1 -4
- data/lib/dynflow/action/rescue.rb +1 -2
- data/lib/dynflow/action/singleton.rb +1 -0
- data/lib/dynflow/action/suspended.rb +1 -0
- data/lib/dynflow/action/timeouts.rb +2 -1
- data/lib/dynflow/action/with_bulk_sub_plans.rb +1 -0
- data/lib/dynflow/action/with_polling_sub_plans.rb +1 -1
- data/lib/dynflow/action/with_sub_plans.rb +20 -19
- data/lib/dynflow/action.rb +37 -37
- data/lib/dynflow/active_job/queue_adapter.rb +2 -1
- data/lib/dynflow/actor.rb +2 -2
- data/lib/dynflow/actors/execution_plan_cleaner.rb +1 -0
- data/lib/dynflow/actors.rb +1 -0
- data/lib/dynflow/clock.rb +3 -4
- data/lib/dynflow/config.rb +6 -5
- data/lib/dynflow/connectors/abstract.rb +11 -10
- data/lib/dynflow/connectors/database.rb +2 -2
- data/lib/dynflow/connectors/direct.rb +2 -3
- data/lib/dynflow/connectors.rb +1 -0
- data/lib/dynflow/coordinator.rb +2 -6
- data/lib/dynflow/coordinator_adapters/abstract.rb +1 -0
- data/lib/dynflow/coordinator_adapters/sequel.rb +1 -0
- data/lib/dynflow/coordinator_adapters.rb +1 -2
- data/lib/dynflow/dead_letter_silencer.rb +1 -0
- data/lib/dynflow/debug/telemetry/persistence.rb +3 -2
- data/lib/dynflow/delayed_executors/abstract.rb +1 -2
- data/lib/dynflow/delayed_executors/abstract_core.rb +1 -1
- data/lib/dynflow/delayed_executors/polling.rb +1 -2
- data/lib/dynflow/delayed_executors.rb +1 -2
- data/lib/dynflow/delayed_plan.rb +6 -6
- data/lib/dynflow/director/execution_plan_manager.rb +1 -1
- data/lib/dynflow/director/flow_manager.rb +1 -0
- data/lib/dynflow/director/queue_hash.rb +2 -1
- data/lib/dynflow/director/running_steps_manager.rb +3 -2
- data/lib/dynflow/director/sequence_cursor.rb +1 -2
- data/lib/dynflow/director/sequential_manager.rb +1 -0
- data/lib/dynflow/director.rb +12 -11
- data/lib/dynflow/dispatcher/abstract.rb +1 -0
- data/lib/dynflow/dispatcher/client_dispatcher.rb +33 -33
- data/lib/dynflow/dispatcher/executor_dispatcher.rb +7 -6
- data/lib/dynflow/dispatcher.rb +8 -7
- data/lib/dynflow/errors.rb +1 -0
- data/lib/dynflow/execution_history.rb +2 -1
- data/lib/dynflow/execution_plan/dependency_graph.rb +1 -2
- data/lib/dynflow/execution_plan/hooks.rb +1 -1
- data/lib/dynflow/execution_plan/output_reference.rb +4 -4
- data/lib/dynflow/execution_plan/steps/abstract.rb +21 -20
- data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb +1 -1
- data/lib/dynflow/execution_plan/steps/error.rb +10 -9
- data/lib/dynflow/execution_plan/steps/finalize_step.rb +1 -2
- data/lib/dynflow/execution_plan/steps/plan_step.rb +12 -11
- data/lib/dynflow/execution_plan/steps/run_step.rb +1 -1
- data/lib/dynflow/execution_plan/steps.rb +1 -2
- data/lib/dynflow/execution_plan.rb +46 -46
- data/lib/dynflow/executors/abstract/core.rb +4 -3
- data/lib/dynflow/executors/parallel/core.rb +3 -2
- data/lib/dynflow/executors/parallel/pool.rb +1 -4
- data/lib/dynflow/executors/parallel/worker.rb +1 -0
- data/lib/dynflow/executors/parallel.rb +3 -2
- data/lib/dynflow/executors/sidekiq/core.rb +3 -1
- data/lib/dynflow/executors/sidekiq/internal_job_base.rb +1 -0
- data/lib/dynflow/executors/sidekiq/orchestrator_jobs.rb +1 -0
- data/lib/dynflow/executors/sidekiq/redis_locking.rb +1 -0
- data/lib/dynflow/executors/sidekiq/serialization.rb +1 -0
- data/lib/dynflow/executors/sidekiq/worker_jobs.rb +1 -0
- data/lib/dynflow/executors.rb +1 -1
- data/lib/dynflow/extensions/msgpack.rb +5 -4
- data/lib/dynflow/extensions.rb +1 -0
- data/lib/dynflow/flows/abstract.rb +1 -1
- data/lib/dynflow/flows/abstract_composed.rb +1 -2
- data/lib/dynflow/flows/atom.rb +1 -2
- data/lib/dynflow/flows/concurrence.rb +1 -1
- data/lib/dynflow/flows/registry.rb +1 -0
- data/lib/dynflow/flows/sequence.rb +1 -1
- data/lib/dynflow/flows.rb +1 -2
- data/lib/dynflow/logger_adapters/abstract.rb +1 -1
- data/lib/dynflow/logger_adapters/delegator.rb +1 -1
- data/lib/dynflow/logger_adapters/formatters/abstract.rb +1 -0
- data/lib/dynflow/logger_adapters/formatters/exception.rb +1 -0
- data/lib/dynflow/logger_adapters/formatters.rb +1 -0
- data/lib/dynflow/logger_adapters/simple.rb +6 -5
- data/lib/dynflow/logger_adapters.rb +1 -0
- data/lib/dynflow/middleware/common/singleton.rb +1 -0
- data/lib/dynflow/middleware/common/transaction.rb +1 -0
- data/lib/dynflow/middleware/register.rb +1 -0
- data/lib/dynflow/middleware/resolver.rb +2 -3
- data/lib/dynflow/middleware/stack.rb +1 -0
- data/lib/dynflow/middleware/world.rb +1 -2
- data/lib/dynflow/middleware.rb +1 -0
- data/lib/dynflow/persistence.rb +4 -5
- data/lib/dynflow/persistence_adapters/abstract.rb +1 -1
- data/lib/dynflow/persistence_adapters/sequel.rb +14 -14
- data/lib/dynflow/persistence_adapters/sequel_migrations/001_initial.rb +2 -1
- data/lib/dynflow/persistence_adapters/sequel_migrations/002_incremental_progress.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/003_parent_action.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/004_coordinator_records.rb +1 -1
- data/lib/dynflow/persistence_adapters/sequel_migrations/005_envelopes.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/006_fix_data_length.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/008_rename_scheduled_plans_to_delayed_plans.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/009_fix_mysql_data_length.rb +1 -1
- data/lib/dynflow/persistence_adapters/sequel_migrations/010_add_execution_plans_label.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/011_placeholder.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/012_add_delayed_plans_serialized_args.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/013_add_action_columns.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/014_add_step_columns.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/015_add_execution_plan_columns.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/016_add_step_queue.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb +37 -30
- data/lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb +1 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/021_create_output_chunks.rb +4 -3
- data/lib/dynflow/persistence_adapters/sequel_migrations/023_sqlite_workarounds.rb +1 -0
- data/lib/dynflow/persistence_adapters.rb +1 -0
- data/lib/dynflow/rails/configuration.rb +5 -4
- data/lib/dynflow/rails/daemon.rb +1 -1
- data/lib/dynflow/rails.rb +3 -2
- data/lib/dynflow/round_robin.rb +2 -2
- data/lib/dynflow/semaphores/abstract.rb +1 -1
- data/lib/dynflow/semaphores/aggregating.rb +1 -2
- data/lib/dynflow/semaphores/dummy.rb +1 -1
- data/lib/dynflow/semaphores/stateful.rb +1 -1
- data/lib/dynflow/semaphores.rb +1 -0
- data/lib/dynflow/serializable.rb +1 -0
- data/lib/dynflow/serializer.rb +2 -2
- data/lib/dynflow/serializers/abstract.rb +1 -2
- data/lib/dynflow/serializers/noop.rb +1 -2
- data/lib/dynflow/serializers.rb +1 -2
- data/lib/dynflow/stateful.rb +1 -0
- data/lib/dynflow/telemetry.rb +11 -10
- data/lib/dynflow/telemetry_adapters/abstract.rb +1 -0
- data/lib/dynflow/telemetry_adapters/dummy.rb +1 -0
- data/lib/dynflow/telemetry_adapters/statsd.rb +2 -1
- data/lib/dynflow/testing/assertions.rb +7 -7
- data/lib/dynflow/testing/dummy_coordinator.rb +1 -0
- data/lib/dynflow/testing/dummy_execution_plan.rb +1 -0
- data/lib/dynflow/testing/dummy_executor.rb +1 -0
- data/lib/dynflow/testing/dummy_planned_action.rb +3 -1
- data/lib/dynflow/testing/dummy_step.rb +1 -0
- data/lib/dynflow/testing/dummy_world.rb +1 -0
- data/lib/dynflow/testing/factories.rb +42 -37
- data/lib/dynflow/testing/in_thread_executor.rb +1 -0
- data/lib/dynflow/testing/in_thread_world.rb +1 -0
- data/lib/dynflow/testing/managed_clock.rb +1 -1
- data/lib/dynflow/testing/mimic.rb +4 -4
- data/lib/dynflow/testing.rb +1 -0
- data/lib/dynflow/throttle_limiter.rb +1 -1
- data/lib/dynflow/transaction_adapters/abstract.rb +1 -0
- data/lib/dynflow/transaction_adapters/active_record.rb +1 -0
- data/lib/dynflow/transaction_adapters/none.rb +1 -0
- data/lib/dynflow/transaction_adapters.rb +1 -2
- data/lib/dynflow/utils/indifferent_hash.rb +7 -1
- data/lib/dynflow/utils/priority_queue.rb +1 -0
- data/lib/dynflow/utils.rb +1 -1
- data/lib/dynflow/version.rb +2 -1
- data/lib/dynflow/watchers/memory_consumption_watcher.rb +1 -1
- data/lib/dynflow/web/console.rb +1 -3
- data/lib/dynflow/web/console_helpers.rb +5 -4
- data/lib/dynflow/web/filtering_helpers.rb +1 -0
- data/lib/dynflow/web/world_helpers.rb +1 -0
- data/lib/dynflow/web.rb +3 -3
- data/lib/dynflow/web_console.rb +1 -0
- data/lib/dynflow/world/invalidation.rb +1 -0
- data/lib/dynflow/world.rb +19 -19
- data/lib/dynflow.rb +3 -6
- data/test/abnormal_states_recovery_test.rb +4 -8
- data/test/action_test.rb +10 -18
- data/test/activejob_adapter_test.rb +2 -2
- data/test/batch_sub_tasks_test.rb +1 -1
- data/test/clock_test.rb +2 -3
- data/test/concurrency_control_test.rb +6 -7
- data/test/coordinator_test.rb +1 -0
- data/test/daemon_test.rb +3 -2
- data/test/dead_letter_silencer_test.rb +2 -1
- data/test/dispatcher_test.rb +4 -5
- data/test/execution_plan_cleaner_test.rb +1 -0
- data/test/execution_plan_hooks_test.rb +1 -0
- data/test/execution_plan_test.rb +10 -32
- data/test/executor_test.rb +20 -37
- data/test/extensions_test.rb +1 -0
- data/test/flows_test.rb +2 -2
- data/test/future_execution_test.rb +2 -3
- data/test/memory_cosumption_watcher_test.rb +1 -0
- data/test/middleware_test.rb +4 -6
- data/test/persistence_test.rb +26 -26
- data/test/redis_locking_test.rb +1 -0
- data/test/rescue_test.rb +3 -11
- data/test/round_robin_test.rb +1 -0
- data/test/semaphores_test.rb +5 -7
- data/test/support/code_workflow_example.rb +11 -28
- data/test/support/dummy_example.rb +20 -19
- data/test/support/middleware_example.rb +2 -8
- data/test/support/rescue_example.rb +1 -14
- data/test/support/test_execution_log.rb +1 -2
- data/test/test_helper.rb +3 -7
- data/test/testing_test.rb +6 -8
- data/test/utils_test.rb +1 -0
- data/test/v2_sub_plans_test.rb +1 -0
- data/test/web_console_test.rb +4 -4
- data/test/world_test.rb +4 -3
- metadata +43 -43
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Dynflow
|
3
4
|
module ExecutionPlan::Steps
|
4
5
|
class PlanStep < Abstract
|
@@ -73,17 +74,17 @@ module Dynflow
|
|
73
74
|
def self.new_from_hash(hash, execution_plan_id, world)
|
74
75
|
check_class_matching hash
|
75
76
|
new execution_plan_id,
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
77
|
+
hash[:id],
|
78
|
+
hash[:state],
|
79
|
+
Action.constantize(hash[:action_class]),
|
80
|
+
hash[:action_id],
|
81
|
+
hash_to_error(hash[:error]),
|
82
|
+
world,
|
83
|
+
string_to_time(hash[:started_at]),
|
84
|
+
string_to_time(hash[:ended_at]),
|
85
|
+
hash[:execution_time],
|
86
|
+
hash[:real_time],
|
87
|
+
hash[:children]
|
87
88
|
end
|
88
89
|
|
89
90
|
def load_action
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Dynflow
|
3
4
|
module ExecutionPlan::Steps
|
4
5
|
class RunStep < AbstractFlowStep
|
5
|
-
|
6
6
|
def self.state_transitions
|
7
7
|
@state_transitions ||= {
|
8
8
|
pending: [:running, :skipped, :error], # :skipped when it cannot be run because it depends on skipping step
|
@@ -1,13 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Dynflow
|
3
4
|
module ExecutionPlan::Steps
|
4
|
-
|
5
5
|
require 'dynflow/execution_plan/steps/error'
|
6
6
|
require 'dynflow/execution_plan/steps/abstract'
|
7
7
|
require 'dynflow/execution_plan/steps/abstract_flow_step'
|
8
8
|
require 'dynflow/execution_plan/steps/plan_step'
|
9
9
|
require 'dynflow/execution_plan/steps/run_step'
|
10
10
|
require 'dynflow/execution_plan/steps/finalize_step'
|
11
|
-
|
12
11
|
end
|
13
12
|
end
|
@@ -1,18 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'securerandom'
|
3
4
|
|
4
5
|
module Dynflow
|
5
6
|
# rubocop:disable Metrics/ClassLength
|
6
7
|
# TODO extract planning logic to an extra class ExecutionPlanner
|
7
8
|
class ExecutionPlan < Serializable
|
8
|
-
|
9
9
|
# a fallback object representing a plan with some corrupted data,
|
10
10
|
# preventing to load the whole plan properly, this can be used for presenting
|
11
11
|
# at least some data and not running into internal server errors
|
12
12
|
class InvalidPlan
|
13
13
|
attr_reader :exception, :id, :label, :state,
|
14
|
-
|
15
|
-
|
14
|
+
:started_at, :ended_at,
|
15
|
+
:execution_time, :real_time, :execution_history
|
16
16
|
|
17
17
|
def initialize(exception, id, label, state,
|
18
18
|
started_at = nil, ended_at = nil,
|
@@ -45,8 +45,8 @@ module Dynflow
|
|
45
45
|
require 'dynflow/execution_plan/dependency_graph'
|
46
46
|
|
47
47
|
attr_reader :id, :world, :label,
|
48
|
-
|
49
|
-
|
48
|
+
:root_plan_step, :steps, :run_flow, :finalize_flow,
|
49
|
+
:started_at, :ended_at, :execution_time, :real_time, :execution_history
|
50
50
|
|
51
51
|
def self.states
|
52
52
|
@states ||= [:pending, :scheduled, :planning, :planned, :running, :paused, :stopped]
|
@@ -83,9 +83,9 @@ module Dynflow
|
|
83
83
|
real_time = 0.0,
|
84
84
|
execution_history = ExecutionHistory.new)
|
85
85
|
id ||= SecureRandom.uuid
|
86
|
-
@id
|
87
|
-
@world
|
88
|
-
@label
|
86
|
+
@id = Type! id, String
|
87
|
+
@world = Type! world, World
|
88
|
+
@label = Type! label, String, NilClass
|
89
89
|
self.state = state
|
90
90
|
@run_flow = Type! run_flow, Flows::Abstract
|
91
91
|
@finalize_flow = Type! finalize_flow, Flows::Abstract
|
@@ -130,7 +130,7 @@ module Dynflow
|
|
130
130
|
key = failure? ? :failure : :success
|
131
131
|
Dynflow::Telemetry.with_instance do |t|
|
132
132
|
t.increment_counter(:dynflow_finished_execution_plans, 1,
|
133
|
-
|
133
|
+
telemetry_common_options.merge(:result => key.to_s))
|
134
134
|
end
|
135
135
|
hooks_to_run << key
|
136
136
|
unlock_all_singleton_locks!
|
@@ -140,11 +140,11 @@ module Dynflow
|
|
140
140
|
# ignore
|
141
141
|
end
|
142
142
|
logger.debug format('%13s %s %9s >> %9s',
|
143
|
-
|
143
|
+
'ExecutionPlan', id, original, state)
|
144
144
|
add_history_notice(history_notice)
|
145
145
|
self.save
|
146
146
|
toggle_telemetry_state original == :pending ? nil : original.to_s,
|
147
|
-
|
147
|
+
self.state == :stopped ? nil : self.state.to_s
|
148
148
|
hooks_to_run.each { |kind| run_hooks kind }
|
149
149
|
end
|
150
150
|
|
@@ -238,7 +238,7 @@ module Dynflow
|
|
238
238
|
end
|
239
239
|
|
240
240
|
def steps_in_state(*states)
|
241
|
-
self.steps.values.find_all {|step| states.include?(step.state) }
|
241
|
+
self.steps.values.find_all { |step| states.include?(step.state) }
|
242
242
|
end
|
243
243
|
|
244
244
|
def generate_action_id
|
@@ -257,11 +257,11 @@ module Dynflow
|
|
257
257
|
run_hooks(:pending)
|
258
258
|
serializer = root_plan_step.delay(delay_options, args)
|
259
259
|
delayed_plan = DelayedPlan.new(@world,
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
260
|
+
id,
|
261
|
+
delay_options[:start_at],
|
262
|
+
delay_options.fetch(:start_before, nil),
|
263
|
+
serializer,
|
264
|
+
delay_options[:frozen] || false)
|
265
265
|
persistence.save_delayed_plan(delayed_plan)
|
266
266
|
ensure
|
267
267
|
update_state(error? ? :stopped : :scheduled)
|
@@ -455,36 +455,36 @@ module Dynflow
|
|
455
455
|
execution_plan_id = hash[:id]
|
456
456
|
steps = steps_from_hash(hash[:step_ids], execution_plan_id, world)
|
457
457
|
self.new(world,
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
458
|
+
execution_plan_id,
|
459
|
+
hash[:label],
|
460
|
+
hash[:state],
|
461
|
+
steps[hash[:root_plan_step_id]],
|
462
|
+
load_flow(hash[:run_flow]),
|
463
|
+
load_flow(hash[:finalize_flow]),
|
464
|
+
steps,
|
465
|
+
string_to_time(hash[:started_at]),
|
466
|
+
string_to_time(hash[:ended_at]),
|
467
|
+
hash[:execution_time].to_f,
|
468
|
+
hash[:real_time].to_f,
|
469
|
+
ExecutionHistory.new_from_hash(hash[:execution_history]))
|
470
470
|
rescue => plan_exception
|
471
471
|
begin
|
472
472
|
world.logger.error("Could not load execution plan #{execution_plan_id}")
|
473
473
|
world.logger.error(plan_exception)
|
474
474
|
InvalidPlan.new(plan_exception, execution_plan_id,
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
475
|
+
hash[:label],
|
476
|
+
hash[:state],
|
477
|
+
string_to_time(hash[:started_at]),
|
478
|
+
string_to_time(hash[:ended_at]),
|
479
|
+
hash[:execution_time].to_f,
|
480
|
+
hash[:real_time].to_f,
|
481
|
+
ExecutionHistory.new_from_hash(hash[:execution_history]))
|
482
482
|
rescue => invalid_plan_exception
|
483
483
|
world.logger.error("Could not even load a fallback execution plan for #{execution_plan_id}")
|
484
484
|
world.logger.error(invalid_plan_exception)
|
485
485
|
InvalidPlan.new(invalid_plan_exception, execution_plan_id,
|
486
|
-
|
487
|
-
|
486
|
+
hash[:label],
|
487
|
+
hash[:state])
|
488
488
|
end
|
489
489
|
end
|
490
490
|
|
@@ -530,12 +530,12 @@ module Dynflow
|
|
530
530
|
|
531
531
|
def add_step(step_class, action_class, action_id, state = :pending)
|
532
532
|
step_class.new(self.id,
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
533
|
+
self.generate_step_id,
|
534
|
+
state,
|
535
|
+
action_class,
|
536
|
+
action_id,
|
537
|
+
nil,
|
538
|
+
world).tap do |new_step|
|
539
539
|
@steps[new_step.id] = new_step
|
540
540
|
end
|
541
541
|
end
|
@@ -571,9 +571,9 @@ module Dynflow
|
|
571
571
|
@label = root_plan_step.action_class if @label.nil?
|
572
572
|
Dynflow::Telemetry.with_instance do |t|
|
573
573
|
t.set_gauge(:dynflow_active_execution_plans, '-1',
|
574
|
-
|
574
|
+
telemetry_common_options.merge(:state => original)) unless original.nil?
|
575
575
|
t.set_gauge(:dynflow_active_execution_plans, '+1',
|
576
|
-
|
576
|
+
telemetry_common_options.merge(:state => new)) unless new.nil?
|
577
577
|
end
|
578
578
|
end
|
579
579
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Dynflow
|
3
4
|
module Executors
|
4
5
|
module Abstract
|
@@ -20,7 +21,7 @@ module Dynflow
|
|
20
21
|
def handle_execution(execution_plan_id, finished)
|
21
22
|
if terminating?
|
22
23
|
raise Dynflow::Error,
|
23
|
-
|
24
|
+
"cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
|
24
25
|
end
|
25
26
|
|
26
27
|
handle_work(@director.start_execution(execution_plan_id, finished))
|
@@ -30,7 +31,7 @@ module Dynflow
|
|
30
31
|
Type! event, Director::Event
|
31
32
|
if terminating?
|
32
33
|
raise Dynflow::Error,
|
33
|
-
|
34
|
+
"cannot accept event: #{event} core is terminating"
|
34
35
|
end
|
35
36
|
handle_work(@director.handle_event(event))
|
36
37
|
end
|
@@ -38,7 +39,7 @@ module Dynflow
|
|
38
39
|
def handle_planning(execution_plan_id)
|
39
40
|
if terminating?
|
40
41
|
raise Dynflow::Error,
|
41
|
-
|
42
|
+
"cannot accept event: #{event} core is terminating"
|
42
43
|
end
|
43
44
|
|
44
45
|
handle_work(@director.handle_planning(execution_plan_id))
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'dynflow/executors/parallel/pool'
|
3
4
|
require 'dynflow/executors/parallel/worker'
|
4
5
|
|
@@ -19,8 +20,8 @@ module Dynflow
|
|
19
20
|
@queues_options.each do |(queue_name, queue_options)|
|
20
21
|
queue_pool_size = queue_options.fetch(:pool_size, default_pool_size)
|
21
22
|
@pools[queue_name] = Pool.spawn("pool #{queue_name}", @world,
|
22
|
-
|
23
|
-
|
23
|
+
reference, queue_name, queue_pool_size,
|
24
|
+
@world.transaction_adapter)
|
24
25
|
end
|
25
26
|
end
|
26
27
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Dynflow
|
3
4
|
module Executors
|
4
5
|
class Parallel
|
@@ -16,10 +17,6 @@ module Dynflow
|
|
16
17
|
@jobs.shift
|
17
18
|
end
|
18
19
|
|
19
|
-
def queue_size
|
20
|
-
execution_status.values.reduce(0, :+)
|
21
|
-
end
|
22
|
-
|
23
20
|
def empty?
|
24
21
|
@jobs.empty?
|
25
22
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Dynflow
|
3
4
|
module Executors
|
4
5
|
class Parallel
|
@@ -12,7 +13,7 @@ module Dynflow
|
|
12
13
|
def initialize(world,
|
13
14
|
executor_class:,
|
14
15
|
heartbeat_interval:,
|
15
|
-
queues_options: { :default => { :pool_size => 5 }})
|
16
|
+
queues_options: { :default => { :pool_size => 5 } })
|
16
17
|
@world = world
|
17
18
|
@logger = world.logger
|
18
19
|
@core = executor_class.spawn name: 'parallel-executor-core',
|
@@ -24,7 +25,7 @@ module Dynflow
|
|
24
25
|
accepted = @core.ask([:handle_execution, execution_plan_id, finished])
|
25
26
|
accepted.value! if wait_for_acceptance
|
26
27
|
finished
|
27
|
-
rescue Concurrent::Actor::ActorTerminated
|
28
|
+
rescue Concurrent::Actor::ActorTerminated
|
28
29
|
dynflow_error = Dynflow::Error.new('executor terminated')
|
29
30
|
finished.reject dynflow_error unless finished.resolved?
|
30
31
|
raise dynflow_error
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'dynflow/executors/sidekiq/serialization'
|
3
4
|
require 'dynflow/executors/sidekiq/internal_job_base'
|
4
5
|
require 'dynflow/executors/sidekiq/orchestrator_jobs'
|
@@ -9,9 +10,10 @@ require 'sidekiq-reliable-fetch'
|
|
9
10
|
Sidekiq.configure_server do |config|
|
10
11
|
# Use semi-reliable fetch
|
11
12
|
# for details see https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/blob/master/README.md
|
12
|
-
config
|
13
|
+
config[:semi_reliable_fetch] = true
|
13
14
|
Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
|
14
15
|
end
|
16
|
+
::Sidekiq.strict_args!(false)
|
15
17
|
|
16
18
|
module Dynflow
|
17
19
|
module Executors
|
data/lib/dynflow/executors.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'msgpack'
|
3
4
|
|
4
5
|
module Dynflow
|
@@ -25,13 +26,13 @@ module Dynflow
|
|
25
26
|
|
26
27
|
::DateTime.include ::Dynflow::Extensions::MsgPack::Time
|
27
28
|
::MessagePack::DefaultFactory.register_type(0x02, DateTime,
|
28
|
-
|
29
|
-
|
29
|
+
packer: ->(datetime) { MessagePack::Time::Packer.(datetime.to_time) },
|
30
|
+
unpacker: ->(payload) { unpacker.(payload).to_datetime })
|
30
31
|
|
31
32
|
::Date.include ::Dynflow::Extensions::MsgPack::Time
|
32
33
|
::MessagePack::DefaultFactory.register_type(0x03, Date,
|
33
|
-
|
34
|
-
|
34
|
+
packer: ->(date) { MessagePack::Time::Packer.(date.to_time) },
|
35
|
+
unpacker: ->(payload) { unpacker.(payload).to_date })
|
35
36
|
rescue LoadError
|
36
37
|
# This is fine
|
37
38
|
nil
|
data/lib/dynflow/extensions.rb
CHANGED
data/lib/dynflow/flows/atom.rb
CHANGED
data/lib/dynflow/flows.rb
CHANGED
@@ -1,15 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'forwardable'
|
3
4
|
|
4
5
|
module Dynflow
|
5
6
|
module Flows
|
6
|
-
|
7
7
|
require 'dynflow/flows/registry'
|
8
8
|
require 'dynflow/flows/abstract'
|
9
9
|
require 'dynflow/flows/atom'
|
10
10
|
require 'dynflow/flows/abstract_composed'
|
11
11
|
require 'dynflow/flows/concurrence'
|
12
12
|
require 'dynflow/flows/sequence'
|
13
|
-
|
14
13
|
end
|
15
14
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'English'
|
3
4
|
|
4
5
|
module Dynflow
|
@@ -28,11 +29,11 @@ module Dynflow
|
|
28
29
|
|
29
30
|
def formatter(severity, datetime, prog_name, msg)
|
30
31
|
format "[%s #%d] %5s -- %s%s\n",
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
32
|
+
datetime.strftime('%Y-%m-%d %H:%M:%S.%L'),
|
33
|
+
$PID,
|
34
|
+
severity,
|
35
|
+
(prog_name ? prog_name + ': ' : ''),
|
36
|
+
msg.to_s
|
36
37
|
end
|
37
38
|
|
38
39
|
class ProgNameWrapper
|
@@ -1,9 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'tsort'
|
3
4
|
|
4
5
|
module Dynflow
|
5
6
|
class Middleware::Resolver
|
6
|
-
|
7
7
|
include TSort
|
8
8
|
include Algebrick::TypeCheck
|
9
9
|
|
@@ -49,7 +49,7 @@ module Dynflow
|
|
49
49
|
middleware_deps.reject! { |dep| !deps.has_key?(dep) }
|
50
50
|
end
|
51
51
|
|
52
|
-
return deps.delete_if {|klass, _| klass.nil? }
|
52
|
+
return deps.delete_if { |klass, _| klass.nil? }
|
53
53
|
end
|
54
54
|
|
55
55
|
def tsort_each_node(&block)
|
@@ -59,6 +59,5 @@ module Dynflow
|
|
59
59
|
def tsort_each_child(node, &block)
|
60
60
|
@deps.fetch(node).each(&block)
|
61
61
|
end
|
62
|
-
|
63
62
|
end
|
64
63
|
end
|