dynflow 0.8.6 → 0.8.7

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YWQyYzY3YTljZmEwZmFjODNjYTQ2ZDcxODhjOTIyNjIzMjU3MWQxOQ==
+    MTRkNzg1OTljZjRkYzMyMzg3OTlmYzc0Y2I1OWI2MTZjNGY4ZmQ1ZA==
   data.tar.gz: !binary |-
-    NmU1MDM2NWM3YTk2NmE2NGExZWRhNzUxNzg3YmNmZTg5M2EzOTQ5Mg==
+    Yzg4NmU2OGNkOWMxM2RmZWUxZWE2ZjQyODYzMWEzNGU4ZGUyNDcyNQ==
 SHA512:
   metadata.gz: !binary |-
-    Nzk0NjlhYTAwNzQ4NzU0NDIwNzFiYWQ2NjliZGQxYzJiODAxZTMwOWM4ZDI0
-    MWNhYTI2NzA2YWE4YTVjZWYwYTY0YmNmZmU2M2I1NWEzYjQ5MTM0ODliYTYw
-    ZWQ4MmY1ZGE0MTgzYzM3ZWRhZjhjMjRlYWVhNTM1MmZiNWNjYWI=
+    ZmNiMjdjYTg5ZmI0NTcwYjcyOTRkMDI4OGRhM2I5ZDc2MmRjMjdjOWUxZmVl
+    ZDQxZDM4MzhiZjk1MjljMjFlNTg2ZTA4Njc5ZTY3YmNjOGRkNzQ4NDA4MDRl
+    NjkzOTJmZjBmZDIxYTFjYjJkNjg1ZDcwMGI0ZWFlMjk2NDQ3OTU=
   data.tar.gz: !binary |-
-    MmViMjEzZDliZjg2ZThkNDgzMjQ1ZWU5MTNlN2ZlM2JkYjMwYzBmMGE1MDhj
-    Yjc3ZWU0ZmYxOGU3ZTA2MzgxMTAxZjdjMmE3YjVlZGUxYmQxNGUyYWZmOGZj
-    ODM0Yjg3ODQ0MTVmMGM4NTNlZGE5ZWJlY2Q3ZGVjMjc2ZDhjMTc=
+    ZTJiZGRjOTY2ODdhZjMzMTBhNDEzMDhjYjQ2ZGZiYmU0MzU5ZGI1ZjBmZDBi
+    ODM4MDJkNDAxYWJkMDhlY2NlMGEzY2RjNzkyNDBlZTI1NmU1MGFkZTMwNDFh
+    ZTQyYzJiODM0Y2Y3NDRlM2M3NmQ0NDhkZmY2MDQ0MDBjOWFmMTc=

data/dynflow.gemspec

@@ -23,12 +23,12 @@ Gem::Specification.new do |s|
   s.add_dependency "algebrick", '~> 0.7.0'
   s.add_dependency "concurrent-ruby", '~> 0.9.0'
   s.add_dependency "concurrent-ruby-edge", '~> 0.1.0'
+  s.add_dependency "sequel"
 
   s.add_development_dependency "rack-test"
   s.add_development_dependency "minitest"
   s.add_development_dependency "minitest-reporters"
   s.add_development_dependency "activerecord"
-  s.add_development_dependency "sequel"
   s.add_development_dependency "sqlite3"
   s.add_development_dependency "sinatra"
 end

data/lib/dynflow/config.rb

@@ -77,7 +77,7 @@ module Dynflow
       !!config.executor
     end
 
-    config_attr :validity_check_timeout, Fixnum do
+    config_attr :validity_check_timeout, Numeric do
       5
     end
 

data/lib/dynflow/coordinator.rb

@@ -155,6 +155,17 @@ module Dynflow
         @data[:world_id]
       end
 
+      def self.valid_owner_ids(coordinator)
+        coordinator.find_worlds.map { |w| "world:#{w.id}" }
+      end
+
+      def self.valid_classes
+        @valid_classes ||= []
+      end
+
+      def self.inherited(klass)
+        valid_classes << klass
+      end
     end
 
     class DelayedExecutorLock < LockByWorld
@@ -277,6 +288,7 @@ module Dynflow
 
     def delete_world(world)
       Type! world, Coordinator::ClientWorld, Coordinator::ExecutorWorld
+      release_by_owner("world:#{world.id}")
       delete_record(world)
     end
 
@@ -285,5 +297,24 @@ module Dynflow
       world.active = false
       update_record(world)
     end
+
+    def clean_orphaned_locks
+      cleanup_classes = [LockByWorld]
+      ret = []
+      cleanup_classes.each do |cleanup_class|
+        valid_owner_ids = cleanup_class.valid_owner_ids(self)
+        valid_classes = cleanup_class.valid_classes.map(&:name)
+        orphaned_locks = find_locks(class: valid_classes, exclude_owner_id: valid_owner_ids)
+        # reloading the valid owner ids to avoid race conditions
+        valid_owner_ids = cleanup_class.valid_owner_ids(self)
+        orphaned_locks.each do |lock|
+          unless valid_owner_ids.include?(lock.owner_id)
+            release(lock)
+            ret << lock
+          end
+        end
+      end
+      return ret
+    end
   end
 end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -177,7 +177,12 @@ module Dynflow
       def find_coordinator_records(options)
         coordinator_feature!
         options = options.dup
-        data_set = filter(:coordinator_record, table(:coordinator_record), options[:filters])
+        filters = (options[:filters] || {}).dup
+        exclude_owner_id = filters.delete(:exclude_owner_id)
+        data_set = filter(:coordinator_record, table(:coordinator_record), filters)
+        if exclude_owner_id
+          data_set = data_set.exclude(:owner_id => exclude_owner_id)
+        end
         data_set.map { |record| load_data(record) }
       end
 

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.8.6'
+  VERSION = '0.8.7'
 end

data/lib/dynflow/world.rb

@@ -34,7 +34,10 @@ module Dynflow
         @executor_dispatcher = spawn_and_wait(Dispatcher::ExecutorDispatcher, "executor-dispatcher", self)
         executor.initialized.wait
       end
-      self.worlds_validity_check if auto_validity_check
+      if auto_validity_check
+        self.worlds_validity_check
+        self.locks_validity_check
+      end
       @delayed_executor         = try_spawn_delayed_executor(config_for_world)
       @meta                     = config_for_world.meta
       @meta['delayed_executor'] = true if @delayed_executor
@@ -229,7 +232,6 @@ module Dynflow
               clock.ask(:terminate!).wait
             end
 
-            coordinator.release_by_owner("world:#{registered_world.id}")
             coordinator.delete_world(registered_world)
             true
           rescue => e
@@ -327,6 +329,16 @@ module Dynflow
       return results
     end
 
+    def locks_validity_check
+      orphaned_locks = coordinator.clean_orphaned_locks
+
+      unless orphaned_locks.empty?
+        logger.error "invalid coordinator locks found and invalidated: #{orphaned_locks.inspect}"
+      end
+
+      return orphaned_locks
+    end
+
     # executes plans that are planned/paused and haven't reported any error yet (usually when no executor
     # was available by the time of planning or terminating)
     def auto_execute

data/test/abnormal_states_recovery_test.rb

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 require_relative 'test_helper'
+require 'ostruct'
 
 module Dynflow
   module ConsistencyCheckTest
@@ -35,8 +36,8 @@ module Dynflow
       let(:persistence_adapter) { WorldFactory.persistence_adapter }
       let(:shared_connector) { Connectors::Direct.new }
       let(:connector) { Proc.new { |world| shared_connector.start_listening(world); shared_connector } }
-      let(:executor_world) { create_world(true) }
-      let(:executor_world_2) { create_world(true) }
+      let(:executor_world) { create_world(true) { |config| config.auto_validity_check = true } }
+      let(:executor_world_2) { create_world(true) { |config| config.auto_validity_check = true } }
       let(:client_world) { create_world(false) }
       let(:client_world_2) { create_world(false) }
 
@@ -176,8 +177,11 @@ module Dynflow
           end
 
           it 'by default, the auto_validity_check is enabled only for executor words' do
-            create_world(false).auto_validity_check.must_equal false
-            create_world(true).auto_validity_check.must_equal true
+            client_world_config = Config::ForWorld.new(Config.new.tap { |c| c.executor = false }, create_world )
+            client_world_config.auto_validity_check.must_equal false
+
+            executor_world_config = Config::ForWorld.new(Config.new.tap { |c| c.executor = lambda { |w, _| Executors::Parallel.new(w) } }, create_world )
+            executor_world_config.auto_validity_check.must_equal true
           end
 
           it 'reports the validation status' do
@@ -210,6 +214,43 @@ module Dynflow
         end
       end
 
+      describe '#coordinator_validity_check' do
+        describe 'the auto_validity_check is enabled' do
+          let :world_with_auto_validity_check do
+            create_world do |config|
+              config.auto_validity_check = true
+            end
+          end
+
+          let(:invalid_lock) { Coordinator::DelayedExecutorLock.new(OpenStruct.new(:id => 'invalid-world-id')) }
+          let(:valid_lock) { Coordinator::AutoExecuteLock.new(client_world) }
+
+          def current_locks
+            client_world.coordinator.find_locks(id: [valid_lock.id, invalid_lock.id])
+          end
+
+          before do
+            client_world.coordinator.acquire(valid_lock)
+            client_world.coordinator.acquire(invalid_lock)
+            current_locks.must_include(valid_lock)
+            current_locks.must_include(invalid_lock)
+          end
+
+          it 'performs the validity check on world creation if auto_validity_check enabled' do
+            world_with_auto_validity_check
+            current_locks.must_include(valid_lock)
+            current_locks.wont_include(invalid_lock)
+          end
+
+          it 'performs the validity check on world creation if auto_validity_check enabled' do
+            invalid_locks = client_world.locks_validity_check
+            current_locks.must_include(valid_lock)
+            current_locks.wont_include(invalid_lock)
+            invalid_locks.must_include(invalid_lock)
+            invalid_locks.wont_include(valid_lock)
+          end
+        end
+      end
     end
   end
 end

data/test/test_helper.rb

@@ -88,6 +88,7 @@ module WorldFactory
     config.coordinator_adapter = coordinator_adapter
     config.delayed_executor    = nil
     config.auto_rescue         = false
+    config.auto_validity_check = false
     config.exit_on_terminate   = false
     config.auto_execute        = false
     config.auto_terminate      = false
@@ -138,13 +139,14 @@ module TestHelpers
   # allows to create the world inside the tests, using the `connector`
   # and `persistence adapter` from the test context: usefull to create
   # multi-world topology for a signle test
-  def create_world(with_executor = true)
+  def create_world(with_executor = true, &block)
     WorldFactory.create_world do |config|
       config.connector = connector
       config.persistence_adapter = persistence_adapter
       unless with_executor
         config.executor = false
       end
+      block.call(config) if block
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 0.8.6
+  version: 0.8.7
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-17 00:00:00.000000000 Z
+date: 2015-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -82,13 +82,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.1.0
 - !ruby/object:Gem::Dependency
-  name: rack-test
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -96,7 +96,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rack-test
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
@@ -110,7 +110,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest-reporters
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
@@ -124,7 +124,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: minitest-reporters
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
@@ -138,7 +138,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sequel
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='