dynflow 1.8.4 → 1.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bf4beeafa28798576c07fd3fcabd039c026bee7457270426152c6a80be5b86a
-  data.tar.gz: 2ff9ad272fa222d27f4103188b98809edbe150e92ce18a573442215d8cc06b57
+  metadata.gz: 7b7e50b6a77767c3cae603ef773f86c0ef218572ab37d4cf20eaa39a7baeb788
+  data.tar.gz: 3a38fce650541ffc1626a9eab628e0f4e2ccebf8a86ff8e25591ef334805293c
 SHA512:
-  metadata.gz: cb049c6b32cf7a84bce0e616c64648ab13fecbb131233f7ae9a64b5308f3961cee00786b2ca0794bb764a2d42b488a4014b52da5d8de632887834230638e8145
-  data.tar.gz: 4dd52c5e4c1e8644be534a0e48209c17a12ee03c358d81fdefd52b884b8d3b96c2f89417e46790544f68fe80a3004d209a12f2d0954393bb6dd65a4e82b6579e
+  metadata.gz: aea5e2c7ef158cf395e10b7d487070772e9b3a58f72feaea97c12b90c7ad680f69c039a0f443a938f160feceb3993d58bb9eadc91e0be933d6cee43172a35e40
+  data.tar.gz: db00fb0a44f91871d6b0a3b39b6c4d8da7ffe3eaa363ee816e1de95aae576991a83c24a62ecb45212626fdb7de4de3342d84c5ca4fb2c7c766d0c5688301320e

data/examples/halt.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative 'example_helper'
+
+example_description = <<DESC
+
+  Halting example
+  ===================
+
+  This example shows, how halting works in Dynflow. It spawns a single action,
+  which in turn spawns a few evented actions and a single action which occupies
+  the executor for a long time.
+
+  Once the halt event is sent, the execution plan is halted, suspended steps
+  stay suspended forever, running steps stay running until they actually finish
+  the current run and the execution state is flipped over to stopped state.
+
+  You can see the details at #{ExampleHelper::DYNFLOW_URL}
+
+DESC
+
+class EventedCounter < Dynflow::Action
+  def run(event = nil)
+    output[:counter] ||= 0
+    output[:counter] += 1
+    action_logger.info "Iteration #{output[:counter]}"
+
+    if output[:counter] < input[:count]
+      plan_event(:tick, 5)
+      suspend
+    end
+    action_logger.info "Done"
+  end
+end
+
+class Sleeper < Dynflow::Action
+  def run
+    sleep input[:time]
+  end
+end
+
+class Wrapper < Dynflow::Action
+  def plan
+    sequence do
+      concurrence do
+        5.times { |i| plan_action(EventedCounter, :count => i + 1) }
+        plan_action Sleeper, :time => 20
+      end
+      plan_self
+    end
+  end
+
+  def run
+    # Noop
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  puts example_description
+
+  ExampleHelper.world.action_logger.level = Logger::DEBUG
+  ExampleHelper.world
+  t = ExampleHelper.world.trigger(Wrapper)
+  Thread.new do
+    sleep 8
+    ExampleHelper.world.halt(t.id)
+  end
+
+  ExampleHelper.run_web_console
+end

data/lib/dynflow/action/v2/with_sub_plans.rb

@@ -102,7 +102,7 @@ module Dynflow::Action::V2
 
     def increase_counts(planned, failed)
       output[:planned_count] += planned + failed
-      output[:failed_count]  = output.fetch(:failed_count, 0) + failed
+      output[:failed_count] = output.fetch(:failed_count, 0) + failed
       output[:pending_count] = output.fetch(:pending_count, 0) + planned
       output[:success_count] ||= 0
     end
@@ -129,12 +129,20 @@ module Dynflow::Action::V2
     end
 
     def recalculate_counts
-      total   = total_count
-      failed  = sub_plans_count('state' => %w(paused stopped), 'result' => %w(error warning))
+      total = total_count
+      if output[:cancelled_timestamp]
+        cancelled_scheduled_plans = sub_plans_count_after(output[:cancelled_timestamp], { 'state' => %w(paused stopped), 'result' => %w(error warning) })
+        cancelled_unscheduled_plans = total_count - output[:planned_count]
+        cancelled = cancelled_unscheduled_plans + cancelled_scheduled_plans
+      else
+        cancelled = cancelled_scheduled_plans = 0
+      end
+      failed = sub_plans_count('state' => %w(paused stopped), 'result' => %w(error warning)) - cancelled_scheduled_plans
       success = sub_plans_count('state' => 'stopped', 'result' => 'success')
-      output.update(:pending_count => total - failed - success,
-                    :failed_count  => failed - output.fetch(:resumed_count, 0),
-                    :success_count => success)
+      output.update(:pending_count => total - failed - success - cancelled_scheduled_plans,
+        :failed_count => failed - output.fetch(:resumed_count, 0),
+        :success_count => success,
+        :cancelled_count => cancelled)
     end
 
     def counts_set?
@@ -142,7 +150,7 @@ module Dynflow::Action::V2
     end
 
     def check_for_errors!
-      raise SubtaskFailedException.new("A sub task failed") if output[:failed_count] > 0
+      raise SubtaskFailedException.new("A sub task failed") if output[:failed_count] + output[:cancelled_count] > 0
     end
 
     # Helper for creating sub plans
@@ -173,6 +181,7 @@ module Dynflow::Action::V2
     def cancel!(force = false)
       # Count the not-yet-planned tasks as cancelled
       output[:cancelled_count] = total_count - output[:planned_count]
+      output[:cancelled_timestamp] ||= Time.now.utc.iso8601 # time in UTC for comparison with UTC times in the database
       on_planning_finished if output[:cancelled_count].positive?
       # Pass the cancel event to running sub plans if they can be cancelled
       sub_plans(:state => 'running').each { |sub_plan| sub_plan.cancel(force) if sub_plan.cancellable? }
@@ -198,7 +207,9 @@ module Dynflow::Action::V2
     end
 
     def remaining_count
-      total_count - output[:cancelled_count] - output[:planned_count]
+      return 0 if output[:cancelled_timestamp]
+
+      total_count - output[:planned_count]
     end
 
     private
@@ -216,5 +227,9 @@ module Dynflow::Action::V2
     def sub_plans_count(filter = {})
       world.persistence.find_execution_plan_counts(filters: sub_plan_filter.merge(filter))
     end
+
+    def sub_plans_count_after(timestamp, filter = {})
+      world.persistence.find_execution_plan_counts_after(timestamp, { filters: sub_plan_filter.merge(filter) })
+    end
   end
 end

data/lib/dynflow/coordinator.rb

@@ -266,6 +266,19 @@ module Dynflow
       end
     end
 
+    class ExecutionInhibitionLock < Lock
+      def initialize(execution_plan_id)
+        super
+        @data[:owner_id] = "execution-plan:#{execution_plan_id}"
+        @data[:execution_plan_id] = execution_plan_id
+        @data[:id] = self.class.lock_id(execution_plan_id)
+      end
+
+      def self.lock_id(execution_plan_id)
+        "execution-plan:#{execution_plan_id}"
+      end
+    end
+
     class ExecutionLock < LockByWorld
       def initialize(world, execution_plan_id, client_world_id, request_id)
         super(world)

data/lib/dynflow/director/execution_plan_manager.rb

@@ -13,6 +13,7 @@ module Dynflow
         @execution_plan        = Type! execution_plan, ExecutionPlan
         @future                = Type! future, Concurrent::Promises::ResolvableFuture
         @running_steps_manager = RunningStepsManager.new(world)
+        @halted                = false
 
         unless [:planned, :paused].include? execution_plan.state
           raise "execution_plan is not in pending or paused state, it's #{execution_plan.state}"
@@ -25,6 +26,11 @@ module Dynflow
         start_run or start_finalize or finish
       end
 
+      def halt
+        @halted = true
+        @running_steps_manager.terminate
+      end
+
       def restart
         @run_manager = nil
         @finalize_manager = nil
@@ -72,7 +78,7 @@ module Dynflow
       end
 
       def done?
-        (!@run_manager || @run_manager.done?) && (!@finalize_manager || @finalize_manager.done?)
+        @halted || (!@run_manager || @run_manager.done?) && (!@finalize_manager || @finalize_manager.done?)
       end
 
       def terminate
@@ -88,6 +94,7 @@ module Dynflow
       def compute_next_from_step(step)
         raise "run manager not set" unless @run_manager
         raise "run manager already done" if @run_manager.done?
+        return [] if @halted
 
         next_steps = @run_manager.what_is_next(step)
         if @run_manager.done?

data/lib/dynflow/director/running_steps_manager.rb

@@ -16,6 +16,7 @@ module Dynflow
         # to handle potential updates of the step object (that is part of the event)
         @events        = QueueHash.new(Integer, Director::Event)
         @events_by_request_id = {}
+        @halted = false
       end
 
       def terminate
@@ -27,6 +28,10 @@ module Dynflow
         end
       end
 
+      def halt
+        @halted = true
+      end
+
       def add(step, work)
         Type! step, ExecutionPlan::Steps::RunStep
         @running_steps[step.id] = step
@@ -84,6 +89,10 @@ module Dynflow
           event.result.reject UnprocessableEvent.new('step is not suspended, it cannot process events')
           return []
         end
+        if @halted
+          event.result.reject UnprocessableEvent.new('execution plan is halted, it cannot receive events')
+          return []
+        end
 
         can_run_event = @work_items.empty?(step.id)
         @events_by_request_id[event.request_id] = event

data/lib/dynflow/director.rb

@@ -246,8 +246,28 @@ module Dynflow
       end
     end
 
+    def halt(event)
+      halt_execution(event.execution_plan_id)
+    end
+
     private
 
+    def halt_execution(execution_plan_id)
+      manager = @execution_plan_managers[execution_plan_id]
+      @logger.warn "Halting execution plan #{execution_plan_id}"
+      return halt_inactive(execution_plan_id) unless manager
+
+      manager.halt
+      finish_manager manager
+    end
+
+    def halt_inactive(execution_plan_id)
+      plan = @world.persistence.load_execution_plan(execution_plan_id)
+      plan.update_state(:stopped)
+    rescue => e
+      @logger.error e
+    end
+
     def unless_done(manager, work_items)
       return [] unless manager
       if manager.done?
@@ -310,6 +330,14 @@ module Dynflow
           "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
       end
 
+      lock_class = Coordinator::ExecutionInhibitionLock
+      filters = { class: lock_class.to_s, owner_id: lock_class.lock_id(execution_plan_id) }
+      if @world.coordinator.find_records(filters).any?
+        halt_execution(execution_plan_id)
+        raise Dynflow::Error,
+          "cannot execute execution_plan_id:#{execution_plan_id} it's execution is inhibited"
+      end
+
       @execution_plan_managers[execution_plan_id] =
         ExecutionPlanManager.new(@world, execution_plan, finished)
     rescue Dynflow::Error => e

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -141,6 +141,10 @@ module Dynflow
              ignore_unknown = event.optional
              find_executor(event.execution_plan_id)
            end),
+          (on ~Halt do |event|
+             executor = find_executor(event.execution_plan_id)
+             executor == Dispatcher::UnknownWorld ? AnyExecutor : executor
+           end),
           (on Ping.(~any, ~any) | Status.(~any, ~any) do |receiver_id, _|
              receiver_id
            end)
@@ -236,7 +240,7 @@ module Dynflow
             (on Execution.(execution_plan_id: ~any) do |uuid|
                @world.persistence.load_execution_plan(uuid)
              end),
-            (on Event | Ping do
+            (on Event | Ping | Halt do
                true
              end)
           @tracked_requests.delete(id).success! resolve_to

data/lib/dynflow/dispatcher/executor_dispatcher.rb

@@ -13,7 +13,8 @@ module Dynflow
           on(Planning) { perform_planning(envelope, envelope.message) },
           on(Execution) { perform_execution(envelope, envelope.message) },
           on(Event)     { perform_event(envelope, envelope.message) },
-          on(Status)    { get_execution_status(envelope, envelope.message) })
+          on(Status)    { get_execution_status(envelope, envelope.message) },
+          on(Halt)      { halt_execution_plan(envelope, envelope.message) })
       end
 
       protected
@@ -52,6 +53,11 @@ module Dynflow
         end
       end
 
+      def halt_execution_plan(envelope, execution_plan_id)
+        @world.executor.halt execution_plan_id
+        respond(envelope, Done)
+      end
+
       def perform_event(envelope, event_request)
         future = on_finish do |f|
           f.then do

data/lib/dynflow/dispatcher.rb

@@ -29,7 +29,11 @@ module Dynflow
                 execution_plan_id: type { variants String, NilClass }
       end
 
-      variants Event, Execution, Ping, Status, Planning
+      Halt = type do
+        fields! execution_plan_id: String, optional: Algebrick::Types::Boolean
+      end
+
+      variants Event, Execution, Ping, Status, Planning, Halt
     end
 
     Response = Algebrick.type do

data/lib/dynflow/execution_plan.rb

@@ -133,7 +133,9 @@ module Dynflow
             telemetry_common_options.merge(:result => key.to_s))
         end
         hooks_to_run << key
+        world.persistence.delete_delayed_plans(:execution_plan_uuid => id) if delay_record && original == :scheduled
         unlock_all_singleton_locks!
+        unlock_execution_inhibition_lock!
       when :paused
         unlock_all_singleton_locks!
       else
@@ -566,6 +568,14 @@ module Dynflow
       end
     end
 
+    def unlock_execution_inhibition_lock!
+      filter = { :owner_id => 'execution-plan:' + self.id,
+                 :class => Dynflow::Coordinator::ExecutionInhibitionLock.to_s }
+      world.coordinator.find_locks(filter).each do |lock|
+        world.coordinator.release(lock)
+      end
+    end
+
     def toggle_telemetry_state(original, new)
       return if original == new
       @label = root_plan_step.action_class if @label.nil?

data/lib/dynflow/executors/abstract/core.rb

@@ -66,6 +66,10 @@ module Dynflow
           end
         end
 
+        def halt(execution_plan_id)
+          @director.halt execution_plan_id
+        end
+
         def start_termination(*args)
           logger.info 'shutting down Core ...'
           super

data/lib/dynflow/executors/parallel.rb

@@ -57,6 +57,10 @@ module Dynflow
         @core.ask!([:execution_status, execution_plan_id])
       end
 
+      def halt(execution_plan_id)
+        @core.tell([:halt, execution_plan_id])
+      end
+
       def initialized
         @core_initialized
       end

data/lib/dynflow/persistence.rb

@@ -73,6 +73,10 @@ module Dynflow
       adapter.find_execution_plan_counts(options)
     end
 
+    def find_execution_plan_counts_after(timestamp, options)
+      adapter.find_execution_plan_counts_after(timestamp, options)
+    end
+
     def delete_execution_plans(filters, batch_size = 1000, enforce_backup_dir = nil)
       backup_dir = enforce_backup_dir || current_backup_dir
       adapter.delete_execution_plans(filters, batch_size, backup_dir)

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -46,6 +46,10 @@ module Dynflow
         filter(:execution_plan, options[:filters]).count
       end
 
+      def find_execution_plan_counts_after(timestamp, options = {})
+        raise NotImplementedError
+      end
+
       def find_execution_plan_statuses(options)
         raise NotImplementedError
       end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -78,6 +78,10 @@ module Dynflow
         filter(:execution_plan, table(:execution_plan), options[:filters]).count
       end
 
+      def find_execution_plan_counts_after(timestamp, options = {})
+        filter(:execution_plan, table(:execution_plan), options[:filters]).filter(::Sequel.lit('ended_at >= ?', timestamp)).count
+      end
+
       def find_execution_plan_statuses(options)
         plans = filter(:execution_plan, table(:execution_plan), options[:filters])
                 .select(:uuid, :state, :result)

data/lib/dynflow/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dynflow
-  VERSION = '1.8.4'
+  VERSION = '1.9.1'
 end

data/lib/dynflow/world/invalidation.rb

@@ -29,12 +29,27 @@ module Dynflow
             end
           end
 
+          prune_execution_inhibition_locks!
+
           pruned = persistence.prune_envelopes(world.id)
           logger.error("Pruned #{pruned} envelopes for invalidated world #{world.id}") unless pruned.zero?
           coordinator.delete_world(world)
         end
       end
 
+      # Prunes execution inhibition locks which got somehow left behind.
+      # Any execution inhibition locks, which have their corresponding execution
+      # plan in stopped state, will be removed.
+      def prune_execution_inhibition_locks!
+        locks = coordinator.find_locks(class: Coordinator::ExecutionInhibitionLock.name)
+        uuids = locks.map { |lock| lock.data[:execution_plan_id] }
+        plan_uuids = persistence.find_execution_plans(filters: { uuid: uuids, state: 'stopped' }).map(&:id)
+
+        locks.select { |lock| plan_uuids.include? lock.data[:execution_plan_id] }.each do |lock|
+          coordinator.release(lock)
+        end
+      end
+
       def invalidate_planning_lock(planning_lock)
         with_valid_execution_plan_for_lock(planning_lock) do |plan|
           plan.steps.values.each { |step| invalidate_step step }

data/lib/dynflow/world.rb

@@ -4,6 +4,7 @@
 require 'dynflow/world/invalidation'
 
 module Dynflow
+  # rubocop:disable Metrics/ClassLength
   class World
     include Algebrick::TypeCheck
     include Algebrick::Matching
@@ -252,6 +253,11 @@ module Dynflow
       publish_request(Dispatcher::Status[world_id, execution_plan_id], done, false, timeout)
     end
 
+    def halt(execution_plan_id, accepted = Concurrent::Promises.resolvable_future)
+      coordinator.acquire(Coordinator::ExecutionInhibitionLock.new(execution_plan_id))
+      publish_request(Dispatcher::Halt[execution_plan_id], accepted, false)
+    end
+
     def publish_request(request, done, wait_for_accepted, timeout = nil)
       accepted = Concurrent::Promises.resolvable_future
       accepted.rescue do |reason|
@@ -390,4 +396,5 @@ module Dynflow
       return actor
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/test/executor_test.rb

@@ -718,6 +718,99 @@ module Dynflow
           assert [world.terminate, world.terminate].map(&:value).all?
         end
       end
+
+      describe 'halting' do
+        include TestHelpers
+        let(:world) { WorldFactory.create_world }
+
+        it 'halts an execution plan with a suspended step' do
+          triggered = world.trigger(Support::DummyExample::PlanEventsAction, ping_time: 1)
+          plan = world.persistence.load_execution_plan(triggered.id)
+          wait_for do
+            plan = world.persistence.load_execution_plan(triggered.id)
+            plan.state == :running
+          end
+          world.halt(triggered.id)
+          wait_for('the execution plan to halt') do
+            plan = world.persistence.load_execution_plan(triggered.id)
+            plan.state == :stopped
+          end
+          _(plan.steps[2].state).must_equal :suspended
+        end
+
+        it 'halts a paused execution plan' do
+          triggered = world.trigger(Support::DummyExample::FailingDummy)
+          plan = world.persistence.load_execution_plan(triggered.id)
+          wait_for do
+            plan = world.persistence.load_execution_plan(triggered.id)
+            plan.state == :paused
+          end
+          world.halt(plan.id)
+          wait_for('the execution plan to halt') do
+            plan = world.persistence.load_execution_plan(triggered.id)
+            plan.state == :stopped
+          end
+          _(plan.steps[2].state).must_equal :error
+        end
+
+        it 'halts a planned execution plan' do
+          plan = world.plan(Support::DummyExample::Dummy)
+          wait_for do
+            plan = world.persistence.load_execution_plan(plan.id)
+            plan.state == :planned
+          end
+          world.halt(plan.id)
+          wait_for('the execution plan to halt') do
+            plan = world.persistence.load_execution_plan(plan.id)
+            plan.state == :stopped
+          end
+          _(plan.steps[2].state).must_equal :pending
+        end
+
+        it 'halts a scheduled execution plan' do
+          plan = world.delay(Support::DummyExample::Dummy, { start_at: Time.now + 120 })
+          wait_for do
+            plan = world.persistence.load_execution_plan(plan.id)
+            plan.state == :scheduled
+          end
+          world.halt(plan.id)
+          wait_for('the execution plan to halt') do
+            plan = world.persistence.load_execution_plan(plan.id)
+            plan.state == :stopped
+          end
+          _(plan.delay_record).must_be :nil?
+          _(plan.steps[1].state).must_equal :pending
+        end
+
+        it 'halts a pending execution plan' do
+          plan = ExecutionPlan.new(world, nil)
+          plan.save
+          world.halt(plan.id)
+          wait_for('the execution plan to halt') do
+            plan = world.persistence.load_execution_plan(plan.id)
+            plan.state == :stopped
+          end
+        end
+      end
+
+      describe 'execution inhibition locks' do
+        include TestHelpers
+        let(:world) { WorldFactory.create_world }
+
+        it 'inhibits execution' do
+          plan = world.plan(Support::DummyExample::Dummy)
+          world.coordinator.acquire(Coordinator::ExecutionInhibitionLock.new(plan.id))
+          triggered = world.execute(plan.id)
+          triggered.wait
+          _(triggered).must_be :rejected?
+
+          plan = world.persistence.load_execution_plan(plan.id)
+          _(plan.state).must_equal :stopped
+
+          locks = world.coordinator.find_locks({ class: Coordinator::ExecutionInhibitionLock.to_s, owner_id: "execution-plan:#{plan.id}" })
+          _(locks).must_be :empty?
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.8.4
+  version: 1.9.1
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-16 00:00:00.000000000 Z
+date: 2025-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: algebrick
@@ -412,6 +412,7 @@ files:
 - examples/clock_benchmark.rb
 - examples/example_helper.rb
 - examples/future_execution.rb
+- examples/halt.rb
 - examples/memory_limit_watcher.rb
 - examples/orchestrate.rb
 - examples/orchestrate_evented.rb
@@ -682,7 +683,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.3.27
 signing_key:
 specification_version: 4
 summary: DYNamic workFLOW engine