dynflow 1.4.2 → 1.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfb6db078bac6d7bb006fa9cdc7ccb30a5eaa507d5c5fde11723c5295fe7990c
-  data.tar.gz: 7959053035ed970863493c066622e51ca16733d17be8869458ef79c6ccfe4244
+  metadata.gz: ffff17987553c70b701f8f7e8b98f5a159c4d0498ccb45b9d41581204edb1b01
+  data.tar.gz: 1399c875ec759cf98a53925caa822e7e03aa8e1e61d9bd2e1957ab69bb07beb8
 SHA512:
-  metadata.gz: 219901624a304846e8d8b3e70e274e2b291923db9f43ae482816b02caa3b967985fc75f936283e5647f98415d788235e7521ac26f28b59dde625084ed9599bf5
-  data.tar.gz: d1a32c183a574de8e3a0aeeb875bafa6082f2a2971a0f8755f441ab5d350cba2de19e2df22c93109335244f8e84ac65b5dc100abd3517c098bc37e7231751ce2
+  metadata.gz: 51f14bfdcae3d32aaae92716a48668f119327cb8873130dbff7ce9551b40f64d0283037c5a77c5fdced78b612ed8046a449c44ec55e1377d56d247a0cedf3256
+  data.tar.gz: 54787c9fe54caa7227c4122470e398891e161618f5bb02f49332ea2e5a54190dd263e3d46a12691fe47218a81d026d67417b501e07785d6e9eda0aa89f4dd916

data/dynflow.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
   s.add_dependency "apipie-params"
   s.add_dependency "algebrick", '~> 0.7.0'
   s.add_dependency "concurrent-ruby", '~> 1.1.3'
-  s.add_dependency "concurrent-ruby-edge", '~> 0.4.1'
+  s.add_dependency "concurrent-ruby-edge", '~> 0.6.0'
   s.add_dependency "sequel", '>= 4.0.0'
 
   s.add_development_dependency "rake"

data/lib/dynflow/action.rb

@@ -352,15 +352,12 @@ module Dynflow
       @step.state = state
     end
 
+    # If this save returns an integer, it means it was an update. The number
+    # represents the number of updated records. If it is 0, then the step was in
+    # an unexpected state and couldn't be updated
     def save_state(conditions = {})
       phase! Executable
-      # If this save returns an integer, it means it was an update. The number
-      #   represents the number of updated records. If it is 0, then the step
-      #   was in an unexpected state and couldn't be updated, in which case we
-      #   raise an exception and crash hard to prevent the step from being
-      #   executed twice
-      count = @step.save(conditions)
-      raise 'Could not save state' if count.kind_of?(Integer) && !count.positive?
+      @step.save(conditions)
     end
 
     def delay(delay_options, *args)
@@ -536,11 +533,11 @@ module Dynflow
     end
 
     # TODO: This is getting out of hand, refactoring needed
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
     def execute_run(event)
       phase! Run
       @world.logger.debug format('%13s %s:%2d got event %s',
                                  'Step', execution_plan_id, @step.id, event) if event
-      @input = OutputReference.dereference @input, world.persistence
 
       case
       when state == :running
@@ -551,8 +548,19 @@ module Dynflow
           raise 'event can be processed only when in suspended state'
         end
 
+        old_state = self.state
         self.state = :running unless self.state == :skipping
-        save_state(:state => %w(pending error skipping suspended))
+        saved = save_state(:state => %w(pending error skipping suspended))
+        if saved.kind_of?(Integer) && !saved.positive?
+          # The step was already in a state we're trying to transition to, most
+          # likely we were about to execute it for the second time after first
+          # execution was forcefully interrupted.
+          # Set error and return to prevent the step from being executed twice
+          set_error "Could not transition step from #{old_state} to #{self.state}, step already in #{self.state}."
+          return
+        end
+
+        @input = OutputReference.dereference @input, world.persistence
         with_error_handling do
           event = Skip if state == :skipping
 
@@ -573,6 +581,7 @@ module Dynflow
         raise "wrong state #{state} when event:#{event}"
       end
     end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
     def execute_finalize
       phase! Finalize

data/lib/dynflow/connectors/abstract.rb

@@ -25,6 +25,10 @@ module Dynflow
         raise NotImplementedError
       end
 
+      def prune_undeliverable_envelopes(world)
+        raise NotImplementedError
+      end
+
       # we need to pass the world, as the connector can be shared
       # between words: we need to know the one to send the message to
       def receive(world, envelope)

data/lib/dynflow/connectors/database.rb

@@ -172,6 +172,10 @@ module Dynflow
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id, :direction => 'outgoing') }
         @core.ask([:handle_envelope, envelope])
       end
+
+      def prune_undeliverable_envelopes(world)
+        world.persistence.prune_undeliverable_envelopes
+      end
     end
   end
 end

data/lib/dynflow/connectors/direct.rb

@@ -68,6 +68,11 @@ module Dynflow
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id) }
         @core.ask([:handle_envelope, envelope])
       end
+
+      def prune_undeliverable_envelopes(_world)
+        # This is a noop
+        0
+      end
     end
   end
 end

data/lib/dynflow/director/running_steps_manager.rb

@@ -20,7 +20,7 @@ module Dynflow
       def terminate
         pending_work = @work_items.clear.values.flatten(1)
         pending_work.each do |w|
-          if EventWorkItem === w
+          if EventWorkItem === w && w.event.result
             w.event.result.reject UnprocessableEvent.new("dropping due to termination")
           end
         end

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -252,7 +252,7 @@ module Dynflow
           future.fulfill(true)
         else
           if @ping_cache.executor?(request.receiver_id)
-            future.reject
+            future.reject false
           else
             yield
           end

data/lib/dynflow/executors.rb

@@ -4,16 +4,38 @@ module Dynflow
 
     require 'dynflow/executors/parallel'
 
-    # Every time we run a code that can be defined outside of Dynflow,
-    # we should wrap it with this method, and we can ensure here to do
-    # necessary cleanup, such as cleaning ActiveRecord connections
-    def self.run_user_code
-      clear_connections = defined?(::ActiveRecord) && ActiveRecord::Base.connected? && ActiveRecord::Base.connection.open_transactions.zero?
-      yield
-    ensure
-      ::ActiveRecord::Base.clear_active_connections! if clear_connections
-      ::Logging.mdc.clear if defined? ::Logging
-    end
+    class << self
+      # Every time we run a code that can be defined outside of Dynflow,
+      # we should wrap it with this method, and we can ensure here to do
+      # necessary cleanup, such as cleaning ActiveRecord connections
+      def run_user_code
+        # Here we cover a case where the connection was already checked out from
+        # the pool and had opened transactions. In that case, we should leave the
+        # cleanup to the other runtime unit which opened the transaction. If the
+        # connection was checked out or there are no opened transactions, we can
+        # safely perform the cleanup.
+        no_previously_opened_transactions = active_record_open_transactions.zero?
+        yield
+      ensure
+        ::ActiveRecord::Base.clear_active_connections! if no_previously_opened_transactions && active_record_connected?
+        ::Logging.mdc.clear if defined? ::Logging
+      end
+
+      private
+
+      def active_record_open_transactions
+        active_record_active_connection&.open_transactions || 0
+      end
 
+      def active_record_active_connection
+        return unless defined?(::ActiveRecord) && ::ActiveRecord::Base.connected?
+        # #active_connection? returns the connection if already established or nil
+        ::ActiveRecord::Base.connection_pool.active_connection?
+      end
+
+      def active_record_connected?
+        !!active_record_active_connection
+      end
+    end
   end
 end

data/lib/dynflow/executors/sidekiq/orchestrator_jobs.rb

@@ -13,7 +13,7 @@ module Dynflow
           def perform(work_item, delayed_events = nil)
             # Usually the step is saved on the worker's side. However if sidekiq is shut down,
             #   then the step may not have been saved so we save it just to be sure
-            if work_item.is_a?(Director::StepWorkItem) && work_item.step&.error&.exception.is_a?(::Sidekiq::Shutdown)
+            if work_item.is_a?(Director::StepWorkItem) && work_item.step&.error&.exception_class == ::Sidekiq::Shutdown
               work_item.step.save
             end
             Dynflow.process_world.executor.core.tell([:work_finished, work_item, delayed_events])

data/lib/dynflow/persistence.rb

@@ -134,5 +134,13 @@ module Dynflow
         envelope
       end
     end
+
+    def prune_envelopes(receiver_ids)
+      adapter.prune_envelopes(receiver_ids)
+    end
+
+    def prune_undeliverable_envelopes
+      adapter.prune_undeliverable_envelopes
+    end
   end
 end

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -116,6 +116,22 @@ module Dynflow
       def push_envelope(envelope)
         raise NotImplementedError
       end
+
+      def prune_envelopes(receiver_ids)
+        raise NotImplementedError
+      end
+
+      def prune_undeliverable_envelopes
+        raise NotImplementedError
+      end
+
+      def migrate_db
+        raise NotImplementedError
+      end
+
+      def abort_if_pending_migrations!
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -45,13 +45,15 @@ module Dynflow
                                step:    %w(error children) }
 
       def initialize(config)
+        migrate = true
         config = config.dup
         @additional_responsibilities = { coordinator: true, connector: true }
-        if config.is_a?(Hash) && config.key?(:additional_responsibilities)
-          @additional_responsibilities.merge!(config.delete(:additional_responsibilities))
+        if config.is_a?(Hash)
+          @additional_responsibilities.merge!(config.delete(:additional_responsibilities)) if config.key?(:additional_responsibilities)
+          migrate = config.fetch(:migrate, true)
         end
         @db = initialize_db config
-        migrate_db
+        migrate_db if migrate
       end
 
       def transaction(&block)
@@ -198,6 +200,16 @@ module Dynflow
         table(:envelope).insert(prepare_record(:envelope, envelope))
       end
 
+      def prune_envelopes(receiver_ids)
+        connector_feature!
+        table(:envelope).where(receiver_id: receiver_ids).delete
+      end
+
+      def prune_undeliverable_envelopes
+        connector_feature!
+        table(:envelope).where(receiver_id: table(:coordinator_record).select(:id)).invert.delete
+      end
+
       def coordinator_feature!
         unless @additional_responsibilities[:coordinator]
           raise "The sequel persistence adapter coordinator feature used but not enabled in additional_features"
@@ -238,6 +250,14 @@ module Dynflow
           envelopes:            table(:envelope).all.to_a }
       end
 
+      def migrate_db
+        ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
+      end
+
+      def abort_if_pending_migrations!
+        ::Sequel::Migrator.check_current(db, self.class.migrations_path, table: 'dynflow_schema_info')
+      end
+
       private
 
       TABLES = { execution_plan:      :dynflow_execution_plans,
@@ -259,10 +279,6 @@ module Dynflow
         File.expand_path('../sequel_migrations', __FILE__)
       end
 
-      def migrate_db
-        ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
-      end
-
       def prepare_record(table_name, value, base = {}, with_data = true)
         record = base.dup
         if with_data && table(table_name).columns.include?(:data)

data/lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+Sequel.migration do
+  up do
+    alter_table(:dynflow_actions) do
+      drop_index [:execution_plan_uuid, :id]
+    end
+
+    alter_table(:dynflow_execution_plans) do
+      drop_index :uuid
+    end
+
+    alter_table(:dynflow_steps) do
+      drop_index [:execution_plan_uuid, :id]
+    end
+  end
+
+  down do
+    alter_table(:dynflow_actions) do
+      add_index [:execution_plan_uuid, :id], :unique => true
+    end
+
+    alter_table(:dynflow_execution_plans) do
+      add_index :uuid, :unique => true
+    end
+
+    alter_table(:dynflow_steps) do
+      add_index [:execution_plan_uuid, :id], :unique => true
+    end
+  end
+end

data/lib/dynflow/rails.rb

@@ -38,8 +38,8 @@ module Dynflow
       init_world.tap do |world|
         @world = world
         config.run_on_init_hooks(false, world)
+        config.increase_db_pool_size(world)
         unless config.remote?
-          config.increase_db_pool_size(world)
           config.run_on_init_hooks(true, world)
           # leave this just for long-running executors
           unless config.rake_task_with_executor?

data/lib/dynflow/rails/configuration.rb

@@ -96,12 +96,23 @@ module Dynflow
       end
 
       def increase_db_pool_size?
-        !::Rails.env.test? && !remote?
+        !::Rails.env.test? && (!remote? || sidekiq_worker?)
+      end
+
+      def sidekiq_worker?
+        defined?(::Sidekiq) && ::Sidekiq.options[:queues].any?
       end
 
       def calculate_db_pool_size(world)
-        self.db_pool_size || world.config.queues.values.inject(5) do |pool_size, pool_options|
-          pool_size += pool_options[:pool_size]
+        return self.db_pool_size if self.db_pool_size
+
+        base_value = 5
+        if defined?(::Sidekiq)
+          Sidekiq.options[:concurrency] + base_value
+        else
+          world.config.queues.values.inject(base_value) do |pool_size, pool_options|
+            pool_size += pool_options[:pool_size]
+          end
         end
       end
 
@@ -185,8 +196,8 @@ module Dynflow
       end
 
       # Sequel adapter based on Rails app database.yml configuration
-      def initialize_persistence(world)
-        persistence_class.new(default_sequel_adapter_options(world))
+      def initialize_persistence(world, options = {})
+        persistence_class.new(default_sequel_adapter_options(world).merge(options))
       end
     end
   end

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.4.2'
+  VERSION = '1.4.7'
 end

data/lib/dynflow/world.rb

@@ -341,7 +341,7 @@ module Dynflow
         @terminating = Concurrent::Promises.future do
           termination_future.wait(termination_timeout)
         end.on_resolution do
-          @terminated.complete
+          @terminated.resolve
           Thread.new { Kernel.exit } if @exit_on_terminate.true?
         end
       end

data/lib/dynflow/world/invalidation.rb

@@ -28,6 +28,8 @@ module Dynflow
             end
           end
 
+          pruned = persistence.prune_envelopes(world.id)
+          logger.error("Pruned #{pruned} envelopes for invalidated world #{world.id}") unless pruned.zero?
           coordinator.delete_world(world)
         end
       end
@@ -41,7 +43,7 @@ module Dynflow
                   else
                     :stopped
                   end
-          plan.update_state(state)
+          plan.update_state(state) if plan.state != state
 
           coordinator.release(planning_lock)
           execute(plan.id) if plan.state == :planned
@@ -115,6 +117,8 @@ module Dynflow
       def perform_validity_checks
         world_invalidation_result = worlds_validity_check
         locks_validity_check
+        pruned = connector.prune_undeliverable_envelopes(self)
+        logger.error("Pruned #{pruned} undeliverable envelopes") unless pruned.zero?
         world_invalidation_result.values.select { |result| result == :invalidated }.size
       end
 

data/test/persistence_test.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require_relative 'test_helper'
 require 'tmpdir'
+require 'ostruct'
 
 module Dynflow
   module PersistenceTest
@@ -371,6 +372,41 @@ module Dynflow
           assert_equal [], adapter.pull_envelopes(executor_world_id)
         end
 
+        it 'supports pruning of envelopes of invalidated worlds' do
+          client_world_id   = '5678'
+          executor_world_id = '1234'
+          envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
+          executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
+          client_envelope   = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
+          envelopes         = [client_envelope, executor_envelope]
+
+          envelopes.each { |e| adapter.push_envelope(e) }
+
+          assert_equal 1, adapter.prune_envelopes([executor_world_id])
+          assert_equal 0, adapter.prune_envelopes([executor_world_id])
+          assert_equal [], adapter.pull_envelopes(executor_world_id)
+          assert_equal [client_envelope], adapter.pull_envelopes(client_world_id)
+        end
+
+        it 'supports pruning of orphaned envelopes' do
+          client_world_id   = '5678'
+          executor_world_id = '1234'
+          envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
+          executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
+          client_envelope   = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
+          envelopes         = [client_envelope, executor_envelope]
+
+          envelopes.each { |e| adapter.push_envelope(e) }
+          adapter.insert_coordinator_record({"class"=>"Dynflow::Coordinator::ExecutorWorld",
+                                             "id" => executor_world_id, "meta" => {}, "active" => true })
+
+          assert_equal 1, adapter.prune_undeliverable_envelopes
+          assert_equal 0, adapter.prune_undeliverable_envelopes
+          assert_equal [], adapter.pull_envelopes(client_world_id)
+          assert_equal [executor_envelope], adapter.pull_envelopes(executor_world_id)
+          assert_equal [], adapter.pull_envelopes(executor_world_id)
+        end
+
         it 'supports reading data saved prior to normalization' do
           db = adapter.send(:db)
           # Prepare records for saving

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.4.7
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-17 00:00:00.000000000 Z
+date: 1970-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -73,14 +73,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -512,6 +512,7 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb
@@ -639,7 +640,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: DYNamic workFLOW engine