dynflow 1.4.3 → 1.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 318bdedfaca8720a62bc3a8bbd7d52b1823e9078ed1b96fecea248a729218335
-  data.tar.gz: 35cfd900ad187608dbefaae809af955a8433a1168929d48ff11e92d164cfc8aa
+  metadata.gz: 541dbb9bd3cd2b7e0deb00b940ac1d387518d2a884263646e6bcc7666ad9a934
+  data.tar.gz: 7741c155b2538ab52c47fd21a2a56bd019bdfd15f5c69e48b4e37044d941bc5d
 SHA512:
-  metadata.gz: 729dd7445b1bae4d6874749539c191912e6468cdcc120340121e9258502f8a6adc9fc7c90d043b3d4ffc6e53392910d75565233f29ce1b48448503aa366b643a
-  data.tar.gz: 9246e6fd8a38383a5e5fd0c17d4dea5b4cc173f03b47c912673db8514cab773ae6c845c47583b85e9f8e833208751273888f73284fb4f90b451d2ed56b02998d
+  metadata.gz: 1965d10b547c1fb091c7dbabeaef50f7825d3b472dc911b5f17d6ee8366c401577234bc6836c85e932c58548bfc6cac82bbce4719ea637793b9815a4feebcb5f
+  data.tar.gz: 66ee5081a627072548a30dedebce8cac285bd6bf0630aa3ddf93305852f6d3daded8d05260c5a896db45345913cb1a0a7a2ef4fa6dbde45aa29b3159ff61a595

data/lib/dynflow/connectors/abstract.rb

@@ -25,6 +25,10 @@ module Dynflow
         raise NotImplementedError
       end
 
+      def prune_undeliverable_envelopes(world)
+        raise NotImplementedError
+      end
+
       # we need to pass the world, as the connector can be shared
       # between words: we need to know the one to send the message to
       def receive(world, envelope)

data/lib/dynflow/connectors/database.rb

@@ -172,6 +172,10 @@ module Dynflow
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id, :direction => 'outgoing') }
         @core.ask([:handle_envelope, envelope])
       end
+
+      def prune_undeliverable_envelopes(world)
+        world.persistence.prune_undeliverable_envelopes
+      end
     end
   end
 end

data/lib/dynflow/connectors/direct.rb

@@ -68,6 +68,11 @@ module Dynflow
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id) }
         @core.ask([:handle_envelope, envelope])
       end
+
+      def prune_undeliverable_envelopes(_world)
+        # This is a noop
+        0
+      end
     end
   end
 end

data/lib/dynflow/executors.rb

@@ -4,16 +4,38 @@ module Dynflow
 
     require 'dynflow/executors/parallel'
 
-    # Every time we run a code that can be defined outside of Dynflow,
-    # we should wrap it with this method, and we can ensure here to do
-    # necessary cleanup, such as cleaning ActiveRecord connections
-    def self.run_user_code
-      clear_connections = defined?(::ActiveRecord) && ActiveRecord::Base.connected? && ActiveRecord::Base.connection.open_transactions.zero?
-      yield
-    ensure
-      ::ActiveRecord::Base.clear_active_connections! if clear_connections
-      ::Logging.mdc.clear if defined? ::Logging
-    end
+    class << self
+      # Every time we run a code that can be defined outside of Dynflow,
+      # we should wrap it with this method, and we can ensure here to do
+      # necessary cleanup, such as cleaning ActiveRecord connections
+      def run_user_code
+        # Here we cover a case where the connection was already checked out from
+        # the pool and had opened transactions. In that case, we should leave the
+        # cleanup to the other runtime unit which opened the transaction. If the
+        # connection was checked out or there are no opened transactions, we can
+        # safely perform the cleanup.
+        no_previously_opened_transactions = active_record_open_transactions.zero?
+        yield
+      ensure
+        ::ActiveRecord::Base.clear_active_connections! if no_previously_opened_transactions && active_record_connected?
+        ::Logging.mdc.clear if defined? ::Logging
+      end
+
+      private
+
+      def active_record_open_transactions
+        active_record_active_connection&.open_transactions || 0
+      end
 
+      def active_record_active_connection
+        return unless defined?(::ActiveRecord) && ::ActiveRecord::Base.connected?
+        # #active_connection? returns the connection if already established or nil
+        ::ActiveRecord::Base.connection_pool.active_connection?
+      end
+
+      def active_record_connected?
+        !!active_record_active_connection
+      end
+    end
   end
 end

data/lib/dynflow/persistence.rb

@@ -134,5 +134,13 @@ module Dynflow
         envelope
       end
     end
+
+    def prune_envelopes(receiver_ids)
+      adapter.prune_envelopes(receiver_ids)
+    end
+
+    def prune_undeliverable_envelopes
+      adapter.prune_undeliverable_envelopes
+    end
   end
 end

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -116,6 +116,22 @@ module Dynflow
       def push_envelope(envelope)
         raise NotImplementedError
       end
+
+      def prune_envelopes(receiver_ids)
+        raise NotImplementedError
+      end
+
+      def prune_undeliverable_envelopes
+        raise NotImplementedError
+      end
+
+      def migrate_db
+        raise NotImplementedError
+      end
+
+      def abort_if_pending_migrations!
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -45,13 +45,15 @@ module Dynflow
                                step:    %w(error children) }
 
       def initialize(config)
+        migrate = true
         config = config.dup
         @additional_responsibilities = { coordinator: true, connector: true }
-        if config.is_a?(Hash) && config.key?(:additional_responsibilities)
-          @additional_responsibilities.merge!(config.delete(:additional_responsibilities))
+        if config.is_a?(Hash)
+          @additional_responsibilities.merge!(config.delete(:additional_responsibilities)) if config.key?(:additional_responsibilities)
+          migrate = config.fetch(:migrate, true)
         end
         @db = initialize_db config
-        migrate_db
+        migrate_db if migrate
       end
 
       def transaction(&block)
@@ -198,6 +200,16 @@ module Dynflow
         table(:envelope).insert(prepare_record(:envelope, envelope))
       end
 
+      def prune_envelopes(receiver_ids)
+        connector_feature!
+        table(:envelope).where(receiver_id: receiver_ids).delete
+      end
+
+      def prune_undeliverable_envelopes
+        connector_feature!
+        table(:envelope).where(receiver_id: table(:coordinator_record).select(:id)).invert.delete
+      end
+
       def coordinator_feature!
         unless @additional_responsibilities[:coordinator]
           raise "The sequel persistence adapter coordinator feature used but not enabled in additional_features"
@@ -238,6 +250,14 @@ module Dynflow
           envelopes:            table(:envelope).all.to_a }
       end
 
+      def migrate_db
+        ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
+      end
+
+      def abort_if_pending_migrations!
+        ::Sequel::Migrator.check_current(db, self.class.migrations_path, table: 'dynflow_schema_info')
+      end
+
       private
 
       TABLES = { execution_plan:      :dynflow_execution_plans,
@@ -259,10 +279,6 @@ module Dynflow
         File.expand_path('../sequel_migrations', __FILE__)
       end
 
-      def migrate_db
-        ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
-      end
-
       def prepare_record(table_name, value, base = {}, with_data = true)
         record = base.dup
         if with_data && table(table_name).columns.include?(:data)

data/lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+Sequel.migration do
+  up do
+    alter_table(:dynflow_actions) do
+      drop_index [:execution_plan_uuid, :id]
+    end
+
+    alter_table(:dynflow_execution_plans) do
+      drop_index :uuid
+    end
+
+    alter_table(:dynflow_steps) do
+      drop_index [:execution_plan_uuid, :id]
+    end
+  end
+
+  down do
+    alter_table(:dynflow_actions) do
+      add_index [:execution_plan_uuid, :id], :unique => true
+    end
+
+    alter_table(:dynflow_execution_plans) do
+      add_index :uuid, :unique => true
+    end
+
+    alter_table(:dynflow_steps) do
+      add_index [:execution_plan_uuid, :id], :unique => true
+    end
+  end
+end

data/lib/dynflow/rails/configuration.rb

@@ -100,8 +100,15 @@ module Dynflow
       end
 
       def calculate_db_pool_size(world)
-        self.db_pool_size || world.config.queues.values.inject(5) do |pool_size, pool_options|
-          pool_size += pool_options[:pool_size]
+        return self.db_pool_size if self.db_pool_size
+
+        base_value = 5
+        if defined?(::Sidekiq)
+          Sidekiq.options[:concurrency] + base_value
+        else
+          world.config.queues.values.inject(base_value) do |pool_size, pool_options|
+            pool_size += pool_options[:pool_size]
+          end
         end
       end
 
@@ -185,8 +192,8 @@ module Dynflow
       end
 
       # Sequel adapter based on Rails app database.yml configuration
-      def initialize_persistence(world)
-        persistence_class.new(default_sequel_adapter_options(world))
+      def initialize_persistence(world, options = {})
+        persistence_class.new(default_sequel_adapter_options(world).merge(options))
       end
     end
   end

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.4.3'
+  VERSION = '1.4.4'
 end

data/lib/dynflow/world/invalidation.rb

@@ -28,6 +28,8 @@ module Dynflow
             end
           end
 
+          pruned = persistence.prune_envelopes(world.id)
+          logger.error("Pruned #{pruned} envelopes for invalidated world #{world.id}") unless pruned.zero?
           coordinator.delete_world(world)
         end
       end
@@ -115,6 +117,8 @@ module Dynflow
       def perform_validity_checks
         world_invalidation_result = worlds_validity_check
         locks_validity_check
+        pruned = connector.prune_undeliverable_envelopes(self)
+        logger.error("Pruned #{pruned} undeliverable envelopes") unless pruned.zero?
         world_invalidation_result.values.select { |result| result == :invalidated }.size
       end
 

data/test/persistence_test.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require_relative 'test_helper'
 require 'tmpdir'
+require 'ostruct'
 
 module Dynflow
   module PersistenceTest
@@ -371,6 +372,41 @@ module Dynflow
           assert_equal [], adapter.pull_envelopes(executor_world_id)
         end
 
+        it 'supports pruning of envelopes of invalidated worlds' do
+          client_world_id   = '5678'
+          executor_world_id = '1234'
+          envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
+          executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
+          client_envelope   = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
+          envelopes         = [client_envelope, executor_envelope]
+
+          envelopes.each { |e| adapter.push_envelope(e) }
+
+          assert_equal 1, adapter.prune_envelopes([executor_world_id])
+          assert_equal 0, adapter.prune_envelopes([executor_world_id])
+          assert_equal [], adapter.pull_envelopes(executor_world_id)
+          assert_equal [client_envelope], adapter.pull_envelopes(client_world_id)
+        end
+
+        it 'supports pruning of orphaned envelopes' do
+          client_world_id   = '5678'
+          executor_world_id = '1234'
+          envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
+          executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
+          client_envelope   = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
+          envelopes         = [client_envelope, executor_envelope]
+
+          envelopes.each { |e| adapter.push_envelope(e) }
+          adapter.insert_coordinator_record({"class"=>"Dynflow::Coordinator::ExecutorWorld",
+                                             "id" => executor_world_id, "meta" => {}, "active" => true })
+
+          assert_equal 1, adapter.prune_undeliverable_envelopes
+          assert_equal 0, adapter.prune_undeliverable_envelopes
+          assert_equal [], adapter.pull_envelopes(client_world_id)
+          assert_equal [executor_envelope], adapter.pull_envelopes(executor_world_id)
+          assert_equal [], adapter.pull_envelopes(executor_world_id)
+        end
+
         it 'supports reading data saved prior to normalization' do
           db = adapter.send(:db)
           # Prepare records for saving

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.4.4
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2020-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -512,6 +512,7 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb