dynflow 1.4.1 → 1.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 727176abc78b46f7b0f0381806561496fba8cdb9e31c5cdea473b5bd6de7bcf0
-  data.tar.gz: aa1d255d09123e50d1d7c718c4738e87fa88741ad23e1f548f0444bdb98fc534
+  metadata.gz: 1d3c22d2470835d38bb89ac45879837ac515268c40f919eb3b57de809cb912f4
+  data.tar.gz: 8552edbffe0f91aa162b5b004d1ef40046f719dee746a2e2cf3da119786ac0d9
 SHA512:
-  metadata.gz: 98f12bd5c1cd06b925601b6c9e6fc5dda7f2f81ecba805251cfb987c9b131bd88be59f57098d8bab4c856b33743815fa2338524ffe3804d4d1dda0e44f73fadc
-  data.tar.gz: 8747b3e08c259a94dd025acb4cfc97654ff8c1ec5782b8bf7876b455187fb789f91d467e1ca9468e48c1a16d745fe3279911652f669b84f4864df8af3066b3cf
+  metadata.gz: cbc4d8031d1836644d0327b2afb83d95f0ada20955e6e5c60bde8de447a2cc83bb2b3bcb116a2e9eb4303ed54bef55ca31c4098d6e11003b3a3717d3f96127ec
+  data.tar.gz: 48db612dd381a90474caac16c524304d15862d8947314abc21dd768b481402dae1c8785377eaf11cc9f9674520ec9061b279312fccc98d5aa932c00cf66284ce

data/.travis.yml

@@ -2,6 +2,7 @@ language: ruby
 
 services:
   - postgresql
+  - redis
 
 rvm:
   - "2.3.1"
@@ -18,6 +19,7 @@ matrix:
       env: "DB=mysql DB_CONN_STRING=mysql2://root@localhost/travis_ci_test"
       services:
         - mysql
+        - redis
     - rvm: "2.4.0"
       env: "DB=sqlite3 DB_CONN_STRING=sqlite:/"
     - rvm: "2.4.0"

data/dynflow.gemspec

@@ -23,13 +23,14 @@ Gem::Specification.new do |s|
   s.add_dependency "apipie-params"
   s.add_dependency "algebrick", '~> 0.7.0'
   s.add_dependency "concurrent-ruby", '~> 1.1.3'
-  s.add_dependency "concurrent-ruby-edge", '~> 0.4.1'
+  s.add_dependency "concurrent-ruby-edge", '~> 0.6.0'
   s.add_dependency "sequel", '>= 4.0.0'
 
   s.add_development_dependency "rake"
   s.add_development_dependency "rack-test"
   s.add_development_dependency "minitest"
   s.add_development_dependency "minitest-reporters"
+  s.add_development_dependency "minitest-stub-const"
   s.add_development_dependency "activerecord"
   s.add_development_dependency 'activejob'
   s.add_development_dependency "sqlite3"

data/lib/dynflow/connectors/abstract.rb

@@ -25,6 +25,10 @@ module Dynflow
         raise NotImplementedError
       end
 
+      def prune_undeliverable_envelopes(world)
+        raise NotImplementedError
+      end
+
       # we need to pass the world, as the connector can be shared
       # between words: we need to know the one to send the message to
       def receive(world, envelope)

data/lib/dynflow/connectors/database.rb

@@ -172,6 +172,10 @@ module Dynflow
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id, :direction => 'outgoing') }
         @core.ask([:handle_envelope, envelope])
       end
+
+      def prune_undeliverable_envelopes(world)
+        world.persistence.prune_undeliverable_envelopes
+      end
     end
   end
 end

data/lib/dynflow/connectors/direct.rb

@@ -68,6 +68,11 @@ module Dynflow
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id) }
         @core.ask([:handle_envelope, envelope])
       end
+
+      def prune_undeliverable_envelopes(_world)
+        # This is a noop
+        0
+      end
     end
   end
 end

data/lib/dynflow/executors.rb

@@ -4,16 +4,38 @@ module Dynflow
 
     require 'dynflow/executors/parallel'
 
-    # Every time we run a code that can be defined outside of Dynflow,
-    # we should wrap it with this method, and we can ensure here to do
-    # necessary cleanup, such as cleaning ActiveRecord connections
-    def self.run_user_code
-      clear_connections = defined?(::ActiveRecord) && ActiveRecord::Base.connected? && ActiveRecord::Base.connection.open_transactions.zero?
-      yield
-    ensure
-      ::ActiveRecord::Base.clear_active_connections! if clear_connections
-      ::Logging.mdc.clear if defined? ::Logging
-    end
+    class << self
+      # Every time we run a code that can be defined outside of Dynflow,
+      # we should wrap it with this method, and we can ensure here to do
+      # necessary cleanup, such as cleaning ActiveRecord connections
+      def run_user_code
+        # Here we cover a case where the connection was already checked out from
+        # the pool and had opened transactions. In that case, we should leave the
+        # cleanup to the other runtime unit which opened the transaction. If the
+        # connection was checked out or there are no opened transactions, we can
+        # safely perform the cleanup.
+        no_previously_opened_transactions = active_record_open_transactions.zero?
+        yield
+      ensure
+        ::ActiveRecord::Base.clear_active_connections! if no_previously_opened_transactions && active_record_connected?
+        ::Logging.mdc.clear if defined? ::Logging
+      end
+
+      private
+
+      def active_record_open_transactions
+        active_record_active_connection&.open_transactions || 0
+      end
 
+      def active_record_active_connection
+        return unless defined?(::ActiveRecord) && ::ActiveRecord::Base.connected?
+        # #active_connection? returns the connection if already established or nil
+        ::ActiveRecord::Base.connection_pool.active_connection?
+      end
+
+      def active_record_connected?
+        !!active_record_active_connection
+      end
+    end
   end
 end

data/lib/dynflow/executors/sidekiq/orchestrator_jobs.rb

@@ -13,7 +13,7 @@ module Dynflow
           def perform(work_item, delayed_events = nil)
             # Usually the step is saved on the worker's side. However if sidekiq is shut down,
             #   then the step may not have been saved so we save it just to be sure
-            if work_item.is_a?(Director::StepWorkItem) && work_item.step&.error&.exception.is_a?(::Sidekiq::Shutdown)
+            if work_item.is_a?(Director::StepWorkItem) && work_item.step&.error&.exception_class == ::Sidekiq::Shutdown
               work_item.step.save
             end
             Dynflow.process_world.executor.core.tell([:work_finished, work_item, delayed_events])

data/lib/dynflow/persistence.rb

@@ -134,5 +134,13 @@ module Dynflow
         envelope
       end
     end
+
+    def prune_envelopes(receiver_ids)
+      adapter.prune_envelopes(receiver_ids)
+    end
+
+    def prune_undeliverable_envelopes
+      adapter.prune_undeliverable_envelopes
+    end
   end
 end

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -116,6 +116,22 @@ module Dynflow
       def push_envelope(envelope)
         raise NotImplementedError
       end
+
+      def prune_envelopes(receiver_ids)
+        raise NotImplementedError
+      end
+
+      def prune_undeliverable_envelopes
+        raise NotImplementedError
+      end
+
+      def migrate_db
+        raise NotImplementedError
+      end
+
+      def abort_if_pending_migrations!
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -45,13 +45,15 @@ module Dynflow
                                step:    %w(error children) }
 
       def initialize(config)
+        migrate = true
         config = config.dup
         @additional_responsibilities = { coordinator: true, connector: true }
-        if config.is_a?(Hash) && config.key?(:additional_responsibilities)
-          @additional_responsibilities.merge!(config.delete(:additional_responsibilities))
+        if config.is_a?(Hash)
+          @additional_responsibilities.merge!(config.delete(:additional_responsibilities)) if config.key?(:additional_responsibilities)
+          migrate = config.fetch(:migrate, true)
         end
         @db = initialize_db config
-        migrate_db
+        migrate_db if migrate
       end
 
       def transaction(&block)
@@ -198,6 +200,16 @@ module Dynflow
         table(:envelope).insert(prepare_record(:envelope, envelope))
       end
 
+      def prune_envelopes(receiver_ids)
+        connector_feature!
+        table(:envelope).where(receiver_id: receiver_ids).delete
+      end
+
+      def prune_undeliverable_envelopes
+        connector_feature!
+        table(:envelope).where(receiver_id: table(:coordinator_record).select(:id)).invert.delete
+      end
+
       def coordinator_feature!
         unless @additional_responsibilities[:coordinator]
           raise "The sequel persistence adapter coordinator feature used but not enabled in additional_features"
@@ -238,6 +250,14 @@ module Dynflow
           envelopes:            table(:envelope).all.to_a }
       end
 
+      def migrate_db
+        ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
+      end
+
+      def abort_if_pending_migrations!
+        ::Sequel::Migrator.check_current(db, self.class.migrations_path, table: 'dynflow_schema_info')
+      end
+
       private
 
       TABLES = { execution_plan:      :dynflow_execution_plans,
@@ -259,10 +279,6 @@ module Dynflow
         File.expand_path('../sequel_migrations', __FILE__)
       end
 
-      def migrate_db
-        ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
-      end
-
       def prepare_record(table_name, value, base = {}, with_data = true)
         record = base.dup
         if with_data && table(table_name).columns.include?(:data)

data/lib/dynflow/persistence_adapters/sequel_migrations/001_initial.rb

@@ -32,7 +32,8 @@ Sequel.migration do
       primary_key [:execution_plan_uuid, :id]
       index [:execution_plan_uuid, :id], :unique => true
       column :action_id, Integer
-      foreign_key [:execution_plan_uuid, :action_id], :dynflow_actions
+      foreign_key [:execution_plan_uuid, :action_id], :dynflow_actions,
+                  name: :dynflow_steps_execution_plan_uuid_fkey1
       index [:execution_plan_uuid, :action_id]
 
       column :data, String, text: true

data/lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+Sequel.migration do
+  up do
+    alter_table(:dynflow_actions) do
+      drop_index [:execution_plan_uuid, :id]
+    end
+
+    alter_table(:dynflow_execution_plans) do
+      drop_index :uuid
+    end
+
+    alter_table(:dynflow_steps) do
+      drop_index [:execution_plan_uuid, :id]
+    end
+  end
+
+  down do
+    alter_table(:dynflow_actions) do
+      add_index [:execution_plan_uuid, :id], :unique => true
+    end
+
+    alter_table(:dynflow_execution_plans) do
+      add_index :uuid, :unique => true
+    end
+
+    alter_table(:dynflow_steps) do
+      add_index [:execution_plan_uuid, :id], :unique => true
+    end
+  end
+end

data/lib/dynflow/rails/configuration.rb

@@ -100,8 +100,15 @@ module Dynflow
       end
 
       def calculate_db_pool_size(world)
-        self.db_pool_size || world.config.queues.values.inject(5) do |pool_size, pool_options|
-          pool_size += pool_options[:pool_size]
+        return self.db_pool_size if self.db_pool_size
+
+        base_value = 5
+        if defined?(::Sidekiq)
+          Sidekiq.options[:concurrency] + base_value
+        else
+          world.config.queues.values.inject(base_value) do |pool_size, pool_options|
+            pool_size += pool_options[:pool_size]
+          end
         end
       end
 
@@ -185,8 +192,8 @@ module Dynflow
       end
 
       # Sequel adapter based on Rails app database.yml configuration
-      def initialize_persistence(world)
-        persistence_class.new(default_sequel_adapter_options(world))
+      def initialize_persistence(world, options = {})
+        persistence_class.new(default_sequel_adapter_options(world).merge(options))
       end
     end
   end

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.4.1'
+  VERSION = '1.4.6'
 end

data/lib/dynflow/world.rb

@@ -341,7 +341,7 @@ module Dynflow
         @terminating = Concurrent::Promises.future do
           termination_future.wait(termination_timeout)
         end.on_resolution do
-          @terminated.complete
+          @terminated.resolve
           Thread.new { Kernel.exit } if @exit_on_terminate.true?
         end
       end

data/lib/dynflow/world/invalidation.rb

@@ -28,6 +28,8 @@ module Dynflow
             end
           end
 
+          pruned = persistence.prune_envelopes(world.id)
+          logger.error("Pruned #{pruned} envelopes for invalidated world #{world.id}") unless pruned.zero?
           coordinator.delete_world(world)
         end
       end
@@ -115,6 +117,8 @@ module Dynflow
       def perform_validity_checks
         world_invalidation_result = worlds_validity_check
         locks_validity_check
+        pruned = connector.prune_undeliverable_envelopes(self)
+        logger.error("Pruned #{pruned} undeliverable envelopes") unless pruned.zero?
         world_invalidation_result.values.select { |result| result == :invalidated }.size
       end
 

data/test/persistence_test.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require_relative 'test_helper'
 require 'tmpdir'
+require 'ostruct'
 
 module Dynflow
   module PersistenceTest
@@ -371,6 +372,41 @@ module Dynflow
           assert_equal [], adapter.pull_envelopes(executor_world_id)
         end
 
+        it 'supports pruning of envelopes of invalidated worlds' do
+          client_world_id   = '5678'
+          executor_world_id = '1234'
+          envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
+          executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
+          client_envelope   = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
+          envelopes         = [client_envelope, executor_envelope]
+
+          envelopes.each { |e| adapter.push_envelope(e) }
+
+          assert_equal 1, adapter.prune_envelopes([executor_world_id])
+          assert_equal 0, adapter.prune_envelopes([executor_world_id])
+          assert_equal [], adapter.pull_envelopes(executor_world_id)
+          assert_equal [client_envelope], adapter.pull_envelopes(client_world_id)
+        end
+
+        it 'supports pruning of orphaned envelopes' do
+          client_world_id   = '5678'
+          executor_world_id = '1234'
+          envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
+          executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
+          client_envelope   = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
+          envelopes         = [client_envelope, executor_envelope]
+
+          envelopes.each { |e| adapter.push_envelope(e) }
+          adapter.insert_coordinator_record({"class"=>"Dynflow::Coordinator::ExecutorWorld",
+                                             "id" => executor_world_id, "meta" => {}, "active" => true })
+
+          assert_equal 1, adapter.prune_undeliverable_envelopes
+          assert_equal 0, adapter.prune_undeliverable_envelopes
+          assert_equal [], adapter.pull_envelopes(client_world_id)
+          assert_equal [executor_envelope], adapter.pull_envelopes(executor_world_id)
+          assert_equal [], adapter.pull_envelopes(executor_world_id)
+        end
+
         it 'supports reading data saved prior to normalization' do
           db = adapter.send(:db)
           # Prepare records for saving

data/test/redis_locking_test.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+require_relative 'test_helper'
+require 'mocha/minitest'
+require 'minitest/stub_const'
+require 'ostruct'
+require 'sidekiq'
+require 'dynflow/executors/sidekiq/core'
+
+module Dynflow
+  module RedisLockingTest
+    describe Executors::Sidekiq::RedisLocking do
+      class Orchestrator
+        include Executors::Sidekiq::RedisLocking
+
+        attr_accessor :logger
+
+        def initialize(world, logger)
+          @world = world
+          @logger = logger
+        end
+      end
+
+      class Logger
+        attr_reader :logs
+        def initialize
+          @logs = []
+        end
+
+        [:info, :error, :fatal].each do |key|
+          define_method key do |message|
+            @logs << [key, message]
+          end
+        end
+      end
+
+      after do
+        ::Sidekiq.redis do |conn|
+          conn.del Executors::Sidekiq::RedisLocking::REDIS_LOCK_KEY
+        end
+      end
+
+      def redis_orchestrator_id
+        ::Sidekiq.redis do |conn|
+          conn.get Executors::Sidekiq::RedisLocking::REDIS_LOCK_KEY
+        end
+      end
+
+      let(:world)  { OpenStruct.new(:id => '12345') }
+      let(:world2) { OpenStruct.new(:id => '67890') }
+      let(:orchestrator)  { Orchestrator.new(world, Logger.new) }
+      let(:orchestrator2) { Orchestrator.new(world2, Logger.new) }
+
+      it 'acquires the lock when it is not taken' do
+        orchestrator.wait_for_orchestrator_lock
+        logs = orchestrator.logger.logs
+        _(redis_orchestrator_id).must_equal world.id
+        _(logs).must_equal [[:info, 'Acquired orchestrator lock, entering active mode.']]
+      end
+
+      it 'reacquires the lock if it was lost' do
+        orchestrator.reacquire_orchestrator_lock
+        logs = orchestrator.logger.logs
+        _(redis_orchestrator_id).must_equal world.id
+        _(logs).must_equal [[:error, 'The orchestrator lock was lost, reacquired']]
+      end
+
+      it 'terminates the process if lock was stolen' do
+        orchestrator.wait_for_orchestrator_lock
+        Process.expects(:kill)
+        orchestrator2.reacquire_orchestrator_lock
+        logs = orchestrator2.logger.logs
+        _(redis_orchestrator_id).must_equal world.id
+        _(logs).must_equal [[:fatal, 'The orchestrator lock was stolen by 12345, aborting.']]
+      end
+
+      it 'polls for the lock availability' do
+        Executors::Sidekiq::RedisLocking.stub_const(:REDIS_LOCK_TTL, 1) do
+          Executors::Sidekiq::RedisLocking.stub_const(:REDIS_LOCK_POLL_INTERVAL, 0.5) do
+            orchestrator.wait_for_orchestrator_lock
+            _(redis_orchestrator_id).must_equal world.id
+            orchestrator2.wait_for_orchestrator_lock
+          end
+        end
+
+        _(redis_orchestrator_id).must_equal world2.id
+        passive, active = orchestrator2.logger.logs
+        _(passive).must_equal [:info, 'Orchestrator lock already taken, entering passive mode.']
+        _(active).must_equal [:info, 'Acquired orchestrator lock, entering active mode.']
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.4.6
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-25 00:00:00.000000000 Z
+date: 2020-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -73,14 +73,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -151,6 +151,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-stub-const
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -498,6 +512,7 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb
@@ -566,6 +581,7 @@ files:
 - test/middleware_test.rb
 - test/persistence_test.rb
 - test/prepare_travis_env.sh
+- test/redis_locking_test.rb
 - test/rescue_test.rb
 - test/round_robin_test.rb
 - test/semaphores_test.rb
@@ -648,6 +664,7 @@ test_files:
 - test/middleware_test.rb
 - test/persistence_test.rb
 - test/prepare_travis_env.sh
+- test/redis_locking_test.rb
 - test/rescue_test.rb
 - test/round_robin_test.rb
 - test/semaphores_test.rb