dynflow 1.2.3 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 403ea79d6ea134104cc54ccaca3edbdce414306b944ccd957f37421ef7070d65
-  data.tar.gz: 7361cc2b7e099d7b537738880dd4197034a7323b9e386c6e937518518931cadc
+  metadata.gz: ef3a326c234f3687c367017a5a64befabfb11075944eed324b5babc8b7adec95
+  data.tar.gz: 615bad06c6a5223613b974e1d7d3ae5c6c9e87d53aaf4d038678ec64168cc498
 SHA512:
-  metadata.gz: 0b8f9367989b27fc3a55294659441c3d0360ccc87d752cebd1f723d5a68400083ec0a0330a29811b868b87b459bf651a1c8fc3bce5c8399556cc26671c0c3d54
-  data.tar.gz: 95248248daebcf5d83b9be6bf3593a1ed55b41d5ef048180d30464091810d36361bdf3bdbc688f58625cdd6f5fdbb2a558fd1d8f56063b21cf65c8f03a8f8458
+  metadata.gz: 1307e77ffb5874d7d7ee4bfc010d85c69de267420e6273790971a54921144114afd9d6c02885213feb8b67fca0de28a8835b149f628330ed7976139842764568
+  data.tar.gz: ceda3668f29ff0aac123d9c767c327dda98822dfdf4d8aca9d6e03df7f28bb7c41a60300f4ee25c4348e2789c1d34cf651d692d196dfe2a0fde4be6c888825be

data/.travis.yml

@@ -2,13 +2,11 @@ sudo: false
 language:
   - ruby
 
-before_install: >-
-  if ruby -v | grep 'ruby 2.2'; then
-    gem install bundler -v '~> 1.17'
-  fi
+services:
+  - mysql
+  - postgresql
 
 rvm:
-  - "2.2.2"
   - "2.3.1"
   - "2.4.0"
   - "2.5.0"

data/Gemfile

@@ -12,26 +12,13 @@ group :pry do
 end
 
 group :postgresql do
-  if RUBY_VERSION <= '2'
-    gem 'pg', '< 0.19'
-  else
-    gem "pg"
-  end
+  gem "pg"
 end
 
 group :mysql do
   gem "mysql2"
 end
 
-if RUBY_VERSION < "2.2.2"
-  gem 'activesupport', '~> 4.2'
-  gem 'sinatra', '~> 1.4.8'
-end
-
-if RUBY_VERSION < '2.3.0'
-  gem 'i18n', '<= 1.5.1'
-end
-
 group :lint do
   gem 'rubocop', '0.39.0'
 end

data/dynflow.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.require_paths = ["lib"]
 
-  s.required_ruby_version = '>= 2.0.0'
+  s.required_ruby_version = '>= 2.3.0'
 
   s.add_dependency "multi_json"
   s.add_dependency "apipie-params"

data/lib/dynflow/actor.rb

@@ -7,10 +7,82 @@ module Dynflow
     end
   end
 
+  # Extend the Concurrent::Actor::Envelope to include information about the origin of the message
+  module EnvelopeBacktraceExtension
+    def initialize(*args)
+      super
+      @origin_backtrace = caller + Actor::BacktraceCollector.current_actor_backtrace
+    end
+
+    def origin_backtrace
+      @origin_backtrace
+    end
+
+    def inspect
+      "#<#{self.class.name}:#{object_id}> @message=#{@message.inspect}, @sender=#{@sender.inspect}, @address=#{@address.inspect}>"
+    end
+  end
+  Concurrent::Actor::Envelope.prepend(EnvelopeBacktraceExtension)
+
   # Common parent for all the Dynflow actors defining some defaults
   # that we preffer here.
   class Actor < Concurrent::Actor::Context
+    module LogWithFullBacktrace
+      def log(level, message = nil, &block)
+        if message.is_a? Exception
+          error = message
+          backtrace = Actor::BacktraceCollector.full_backtrace(error.backtrace)
+          log(level, format("%s (%s)\n%s", error.message, error.class, backtrace.join("\n")))
+        else
+          super
+        end
+      end
+    end
+
+    class SetResultsWithOriginLogging < Concurrent::Actor::Behaviour::SetResults
+      include LogWithFullBacktrace
+
+      def on_envelope(envelope)
+        Actor::BacktraceCollector.with_backtrace(envelope.origin_backtrace) { super }
+      end
+    end
+
+    class BacktraceCollector
+      CONCURRENT_RUBY_LINE = '[ concurrent-ruby ]'.freeze
+
+      class << self
+        def with_backtrace(backtrace)
+          previous_actor_backtrace = Thread.current[:_dynflow_actor_backtrace]
+          Thread.current[:_dynflow_actor_backtrace] = backtrace
+          yield
+        ensure
+          Thread.current[:_dynflow_actor_backtrace] = previous_actor_backtrace
+        end
+
+        def current_actor_backtrace
+          Thread.current[:_dynflow_actor_backtrace] || []
+        end
+
+        def full_backtrace(backtrace)
+          filter_backtrace((backtrace || []) + current_actor_backtrace)
+        end
+
+        private
+
+        def filter_line?(line)
+          %w[concurrent-ruby gems/logging actor.rb].any? { |pattern| line.include?(pattern) }
+        end
+
+        # takes an array of backtrace lines and replaces each chunk
+        def filter_backtrace(backtrace)
+          backtrace.map { |line| filter_line?(line) ? CONCURRENT_RUBY_LINE : line }
+            .chunk_while { |l1, l2| l1.equal?(CONCURRENT_RUBY_LINE) && l2.equal?(CONCURRENT_RUBY_LINE) }
+            .map(&:first)
+        end
+      end
+    end
 
+    include LogWithFullBacktrace
     include MethodicActor
 
     # Behaviour that watches for polite asking for termination
@@ -46,7 +118,7 @@ module Dynflow
     def behaviour_definition
       [*Concurrent::Actor::Behaviour.base(:just_log),
        Concurrent::Actor::Behaviour::Buffer,
-       [Concurrent::Actor::Behaviour::SetResults, :just_log],
+       [SetResultsWithOriginLogging, :just_log],
        Concurrent::Actor::Behaviour::Awaits,
        PoliteTermination,
        Concurrent::Actor::Behaviour::ExecutesContext,

data/lib/dynflow/coordinator.rb

@@ -294,6 +294,22 @@ module Dynflow
       end
     end
 
+    class PlanningLock < LockByWorld
+      def initialize(world, execution_plan_id)
+        super(world)
+        @data.merge!(id: self.class.lock_id(execution_plan_id),
+                     execution_plan_id: execution_plan_id)
+      end
+
+      def self.lock_id(execution_plan_id)
+        'execution-plan:' + execution_plan_id
+      end
+
+      def execution_plan_id
+        @data[:execution_plan_id]
+      end
+    end
+
     attr_reader :adapter
 
     def initialize(coordinator_adapter)

data/lib/dynflow/logger_adapters/formatters/exception.rb

@@ -4,7 +4,8 @@ module Dynflow
       class Exception < Abstract
         def format(message)
           if ::Exception === message
-            "#{message.message} (#{message.class})\n#{message.backtrace.join("\n")}"
+            backtrace = Actor::BacktraceCollector.full_backtrace(message.backtrace)
+            "#{message.message} (#{message.class})\n#{backtrace.join("\n")}"
           else
             message
           end

data/lib/dynflow/rails/daemon.rb

@@ -104,6 +104,7 @@ module Dynflow
           :log_output => true,
           :log_output_syslog => true,
           :monitor_interval => [options[:memory_polling_interval] / 2, 30].min,
+          :force_kill_waittime => options[:force_kill_waittime].try(:to_i),
           :ARGV => [command]
         }
       end
@@ -123,7 +124,8 @@ module Dynflow
                           ENV['EXECUTOR_MEMORY_LIMIT'].to_i
                         end,
           memory_init_delay: (ENV['EXECUTOR_MEMORY_MONITOR_DELAY'] || 7200).to_i, # 2 hours
-          memory_polling_interval: (ENV['EXECUTOR_MEMORY_MONITOR_INTERVAL'] || 60).to_i
+          memory_polling_interval: (ENV['EXECUTOR_MEMORY_MONITOR_INTERVAL'] || 60).to_i,
+          force_kill_waittime: (ENV['EXECUTOR_FORCE_KILL_WAITTIME'] || 60).to_i
         }
       end
 

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '1.2.3'.freeze
+  VERSION = '1.3.0'.freeze
 end

data/lib/dynflow/world.rb

@@ -200,8 +200,10 @@ module Dynflow
 
     def plan_with_options(action_class:, args:, id: nil, caller_action: nil)
       ExecutionPlan.new(self, id).tap do |execution_plan|
-        execution_plan.prepare(action_class, caller_action: caller_action)
-        execution_plan.plan(*args)
+        coordinator.acquire(Coordinator::PlanningLock.new(self, execution_plan.id)) do
+          execution_plan.prepare(action_class, caller_action: caller_action)
+          execution_plan.plan(*args)
+        end
       end
     end
 

data/lib/dynflow/world/invalidation.rb

@@ -11,6 +11,11 @@ module Dynflow
         Type! world, Coordinator::ClientWorld, Coordinator::ExecutorWorld
 
         coordinator.acquire(Coordinator::WorldInvalidationLock.new(self, world)) do
+          coordinator.find_locks(class: Coordinator::PlanningLock.name,
+                                 owner_id: 'world:' + world.id).each do |lock|
+            invalidate_planning_lock lock
+          end
+
           if world.is_a? Coordinator::ExecutorWorld
             old_execution_locks = coordinator.find_locks(class: Coordinator::ExecutionLock.name,
                                                          owner_id: "world:#{world.id}")
@@ -26,6 +31,17 @@ module Dynflow
         end
       end
 
+      def invalidate_planning_lock(planning_lock)
+        with_valid_execution_plan_for_lock(planning_lock) do |plan|
+          plan.steps.values.each { |step| invalidate_step step }
+
+          state = plan.plan_steps.all? { |step| step.state == :success } ? :planned : :stopped
+          plan.update_state(state)
+          coordinator.release(planning_lock)
+          execute(plan.id) if plan.state == :planned
+        end
+      end
+
       # Invalidate an execution lock, left behind by a executor that
       # was executing an execution plan when it was terminated.
       #
@@ -33,14 +49,7 @@ module Dynflow
       # @return [void]
       def invalidate_execution_lock(execution_lock)
         with_valid_execution_plan_for_lock(execution_lock) do |plan|
-          plan.steps.values.each do |step|
-            if step.state == :running
-              step.error = ExecutionPlan::Steps::Error.new("Abnormal termination (previous state: #{step.state})")
-              step.state = :error
-              step.save
-            end
-          end
-
+          plan.steps.values.each { |step| invalidate_step step }
           plan.execution_history.add('terminate execution', execution_lock.world_id)
           plan.update_state(:paused, history_notice: false) if plan.state == :running
           plan.save
@@ -155,6 +164,16 @@ module Dynflow
 
         return orphaned_locks
       end
+
+      private
+
+      def invalidate_step(step)
+        if step.state == :running
+          step.error = ExecutionPlan::Steps::Error.new("Abnormal termination (previous state: #{step.state})")
+          step.state = :error
+          step.save
+        end
+      end
     end
   end
 end

data/test/abnormal_states_recovery_test.rb

@@ -136,6 +136,52 @@ module Dynflow
                               "unlock world-invalidation:#{executor_world.id}"]
             client_world.coordinator.adapter.lock_log.must_equal(expected_locks)
           end
+
+          describe 'planning locks' do
+            it 'releases orphaned planning locks and executes associated execution plans' do
+              plan = client_world.plan(Support::DummyExample::Dummy)
+              plan.set_state(:planning, true)
+              plan.save
+              client_world.coordinator.acquire Coordinator::PlanningLock.new(client_world, plan.id)
+              executor_world.invalidate(client_world.registered_world)
+              expected_locks = ["lock world-invalidation:#{client_world.id}",
+                                "unlock execution-plan:#{plan.id}", # planning lock
+                                "lock execution-plan:#{plan.id}", # execution lock
+                                "unlock world-invalidation:#{client_world.id}"]
+              executor_world.coordinator.adapter.lock_log.must_equal(expected_locks)
+              wait_for do
+                plan = client_world_2.persistence.load_execution_plan(plan.id)
+                plan.state == :stopped
+              end
+            end
+
+            it 'releases orphaned planning locks and stops associated execution plans which did not finish planning' do
+              plan = client_world.plan(Support::DummyExample::Dummy)
+              plan.set_state(:planning, true)
+              plan.save
+              step = plan.plan_steps.first
+              step.set_state(:pending, true)
+              step.save
+              client_world.coordinator.acquire Coordinator::PlanningLock.new(client_world, plan.id)
+              client_world_2.invalidate(client_world.registered_world)
+              expected_locks = ["lock world-invalidation:#{client_world.id}",
+                                "unlock execution-plan:#{plan.id}",
+                                "unlock world-invalidation:#{client_world.id}"]
+              client_world_2.coordinator.adapter.lock_log.must_equal(expected_locks)
+              plan = client_world_2.persistence.load_execution_plan(plan.id)
+              plan.state.must_equal :stopped
+            end
+
+            it 'releases orphaned planning locks without execution plans' do
+              uuid = SecureRandom.uuid
+              client_world.coordinator.acquire Coordinator::PlanningLock.new(client_world, uuid)
+              client_world_2.invalidate(client_world.registered_world)
+              expected_locks = ["lock world-invalidation:#{client_world.id}",
+                                "unlock execution-plan:#{uuid}",
+                                "unlock world-invalidation:#{client_world.id}"]
+              client_world_2.coordinator.adapter.lock_log.must_equal(expected_locks)
+            end
+          end
         end
       end
 

data/test/daemon_test.rb

@@ -85,7 +85,8 @@ class DaemonTest < ActiveSupport::TestCase
     @daemon.expects(:run).twice.with do |_folder, options|
       options[:memory_limit] == 1000 &&
         options[:memory_init_delay] == 100 &&
-        options[:memory_polling_interval] == 200
+        options[:memory_polling_interval] == 200 &&
+        options[:force_kill_waittime] == 40
     end
     @daemons.expects(:run_proc).twice.yields
 
@@ -94,7 +95,8 @@ class DaemonTest < ActiveSupport::TestCase
       executors_count: 2,
       memory_limit: 1000,
       memory_init_delay: 100,
-      memory_polling_interval: 200
+      memory_polling_interval: 200,
+      force_kill_waittime: 40
     )
   end
 
@@ -103,6 +105,7 @@ class DaemonTest < ActiveSupport::TestCase
     ENV['EXECUTOR_MEMORY_LIMIT'] = '1gb'
     ENV['EXECUTOR_MEMORY_MONITOR_DELAY'] = '3'
     ENV['EXECUTOR_MEMORY_MONITOR_INTERVAL'] = '4'
+    ENV['EXECUTOR_FORCE_KILL_WAITTIME'] = '40'
 
     actual = @daemon.send(:default_options)
 
@@ -110,5 +113,6 @@ class DaemonTest < ActiveSupport::TestCase
     assert_equal 1.gigabytes, actual[:memory_limit]
     assert_equal 3, actual[:memory_init_delay]
     assert_equal 4, actual[:memory_polling_interval]
+    assert_equal 40, actual[:force_kill_waittime]
   end
 end

data/test/executor_test.rb

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 require_relative 'test_helper'
+require 'mocha/minitest'
 
 module Dynflow
   module ExecutorTest
@@ -669,6 +670,7 @@ module Dynflow
 
       it 'does not accept new work' do
         assert world.terminate.wait
+        ::Dynflow::Coordinator::PlanningLock.any_instance.stubs(:validate!)
         result = world.trigger(Support::DummyExample::Slow, 0.02)
         result.must_be :planned?
         result.finished.wait

data/web/views/flow_step.erb

@@ -23,6 +23,7 @@
 
 <div class="action">
   <% unless @plan.state == :pending %>
+    <p><b>Queue:</b> <%= step.queue %></p>
     <p><b>Started at:</b> <%= h(step.started_at) %></p>
     <p><b>Ended at:</b> <%= h(step.ended_at) %></p>
     <p><b>Real time:</b> <%= duration_to_s(step.real_time) %></p>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.3.0
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-11 00:00:00.000000000 Z
+date: 2019-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -608,15 +608,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: DYNamic workFLOW engine