dynflow 1.4.3 → 1.5.0

data/lib/dynflow/dispatcher/executor_dispatcher.rb

@@ -52,12 +52,14 @@ module Dynflow
           end
         end
         if event_request.time.nil? || event_request.time < Time.now
-          @world.executor.event(envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, future)
+          @world.executor.event(envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, future,
+                                optional: event_request.optional)
         else
           @world.clock.ping(
             @world.executor,
             event_request.time,
-            Director::Event[envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, Concurrent::Promises.resolvable_future],
+            Director::Event[envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, Concurrent::Promises.resolvable_future,
+                            event_request.optional],
             :delayed_event
           )
           # resolves the future right away - currently we do not wait for the clock ping

data/lib/dynflow/execution_history.rb

@@ -12,7 +12,7 @@ module Dynflow
 
     module Event
       def inspect
-        "#{Time.at(time).utc}: #{name}".tap { |s| s << " @ #{world_id}" if world_id }
+        ["#{Time.at(time).utc}: #{name}", world_id].compact.join(' @ ')
       end
     end
 

data/lib/dynflow/execution_plan.rb

@@ -418,6 +418,14 @@ module Dynflow
       end
     end
 
+    def self.load_flow(flow_hash)
+      if flow_hash.is_a? Hash
+        Flows::Abstract.from_hash(flow_hash)
+      else
+        Flows::Abstract.decode(flow_hash)
+      end
+    end
+
     def to_hash
       recursive_to_hash id:                id,
                         class:             self.class.to_s,
@@ -425,8 +433,8 @@ module Dynflow
                         state:             state,
                         result:            result,
                         root_plan_step_id: root_plan_step && root_plan_step.id,
-                        run_flow:          run_flow,
-                        finalize_flow:     finalize_flow,
+                        run_flow:          run_flow.encode,
+                        finalize_flow:     finalize_flow.encode,
                         step_ids:          steps.map { |id, _| id },
                         started_at:        time_to_str(started_at),
                         ended_at:          time_to_str(ended_at),
@@ -448,8 +456,8 @@ module Dynflow
                hash[:label],
                hash[:state],
                steps[hash[:root_plan_step_id]],
-               Flows::Abstract.from_hash(hash[:run_flow]),
-               Flows::Abstract.from_hash(hash[:finalize_flow]),
+               load_flow(hash[:run_flow]),
+               load_flow(hash[:finalize_flow]),
                steps,
                string_to_time(hash[:started_at]),
                string_to_time(hash[:ended_at]),

data/lib/dynflow/executors.rb

@@ -4,16 +4,38 @@ module Dynflow
 
     require 'dynflow/executors/parallel'
 
-    # Every time we run a code that can be defined outside of Dynflow,
-    # we should wrap it with this method, and we can ensure here to do
-    # necessary cleanup, such as cleaning ActiveRecord connections
-    def self.run_user_code
-      clear_connections = defined?(::ActiveRecord) && ActiveRecord::Base.connected? && ActiveRecord::Base.connection.open_transactions.zero?
-      yield
-    ensure
-      ::ActiveRecord::Base.clear_active_connections! if clear_connections
-      ::Logging.mdc.clear if defined? ::Logging
-    end
+    class << self
+      # Every time we run a code that can be defined outside of Dynflow,
+      # we should wrap it with this method, and we can ensure here to do
+      # necessary cleanup, such as cleaning ActiveRecord connections
+      def run_user_code
+        # Here we cover a case where the connection was already checked out from
+        # the pool and had opened transactions. In that case, we should leave the
+        # cleanup to the other runtime unit which opened the transaction. If the
+        # connection was checked out or there are no opened transactions, we can
+        # safely perform the cleanup.
+        no_previously_opened_transactions = active_record_open_transactions.zero?
+        yield
+      ensure
+        ::ActiveRecord::Base.clear_active_connections! if no_previously_opened_transactions && active_record_connected?
+        ::Logging.mdc.clear if defined? ::Logging
+      end
+
+      private
+
+      def active_record_open_transactions
+        active_record_active_connection&.open_transactions || 0
+      end
 
+      def active_record_active_connection
+        return unless defined?(::ActiveRecord) && ::ActiveRecord::Base.connected?
+        # #active_connection? returns the connection if already established or nil
+        ::ActiveRecord::Base.connection_pool.active_connection?
+      end
+
+      def active_record_connected?
+        !!active_record_active_connection
+      end
+    end
   end
 end

data/lib/dynflow/executors/abstract/core.rb

@@ -37,7 +37,7 @@ module Dynflow
 
         def plan_events(delayed_events)
           delayed_events.each do |event|
-            @world.plan_event(event.execution_plan_id, event.step_id, event.event, event.time)
+            @world.plan_event(event.execution_plan_id, event.step_id, event.event, event.time, optional: event.optional)
           end
         end
 

data/lib/dynflow/executors/parallel.rb

@@ -33,8 +33,8 @@ module Dynflow
         raise e
       end
 
-      def event(request_id, execution_plan_id, step_id, event, future = nil)
-        @core.ask([:handle_event, Director::Event[request_id, execution_plan_id, step_id, event, future]])
+      def event(request_id, execution_plan_id, step_id, event, future = nil, optional: false)
+        @core.ask([:handle_event, Director::Event[request_id, execution_plan_id, step_id, event, future, optional]])
         future
       end
 

data/lib/dynflow/executors/sidekiq/orchestrator_jobs.rb

@@ -13,7 +13,7 @@ module Dynflow
           def perform(work_item, delayed_events = nil)
             # Usually the step is saved on the worker's side. However if sidekiq is shut down,
             #   then the step may not have been saved so we save it just to be sure
-            if work_item.is_a?(Director::StepWorkItem) && work_item.step&.error&.exception.is_a?(::Sidekiq::Shutdown)
+            if work_item.is_a?(Director::StepWorkItem) && work_item.step&.error&.exception_class == ::Sidekiq::Shutdown
               work_item.step.save
             end
             Dynflow.process_world.executor.core.tell([:work_finished, work_item, delayed_events])

data/lib/dynflow/flows.rb

@@ -4,6 +4,7 @@ require 'forwardable'
 module Dynflow
   module Flows
 
+    require 'dynflow/flows/registry'
     require 'dynflow/flows/abstract'
     require 'dynflow/flows/atom'
     require 'dynflow/flows/abstract_composed'

data/lib/dynflow/flows/abstract.rb

@@ -32,6 +32,20 @@ module Dynflow
       def flatten!
         raise NotImplementedError
       end
+
+      def self.new_from_hash(hash)
+        check_class_matching hash
+        new(hash[:flows].map { |flow_hash| from_hash(flow_hash) })
+      end
+
+      def self.decode(data)
+        if data.is_a? Integer
+          Flows::Atom.new(data)
+        else
+          kind, *subflows = data
+          Registry.decode(kind).new(subflows.map { |subflow| self.decode(subflow) })
+        end
+      end
     end
   end
 end

data/lib/dynflow/flows/abstract_composed.rb

@@ -11,8 +11,8 @@ module Dynflow
         @flows = flows
       end
 
-      def to_hash
-        super.merge recursive_to_hash(:flows => flows)
+      def encode
+        [Registry.encode(self)] + flows.map(&:encode)
       end
 
       def <<(v)
@@ -61,11 +61,6 @@ module Dynflow
 
       protected
 
-      def self.new_from_hash(hash)
-        check_class_matching hash
-        new(hash[:flows].map { |flow_hash| from_hash(flow_hash) })
-      end
-
       # adds the +new_flow+ in a way that it's in sequence with
       # the +satisfying_flows+
       def add_to_sequence(satisfying_flows, new_flow)

data/lib/dynflow/flows/atom.rb

@@ -5,8 +5,8 @@ module Dynflow
 
       attr_reader :step_id
 
-      def to_hash
-        super.merge(:step_id => step_id)
+      def encode
+        step_id
       end
 
       def initialize(step_id)

data/lib/dynflow/flows/concurrence.rb

@@ -25,5 +25,7 @@ module Dynflow
         return Concurrence.new(extracted_sub_flows)
       end
     end
+
+    Registry.register!(Concurrence, 'C')
   end
 end

data/lib/dynflow/flows/registry.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+module Dynflow
+  module Flows
+    class Registry
+      class IdentifierTaken < ArgumentError; end
+      class UnknownIdentifier < ArgumentError; end
+
+      class << self
+        def register!(klass, identifier)
+          if (found = serialization_map[identifier])
+            raise IdentifierTaken, "Error setting up mapping #{identifier} to #{klass}, it already maps to #{found}"
+          else
+            serialization_map.update(identifier => klass)
+          end
+        end
+
+        def encode(klass)
+          klass = klass.class unless klass.is_a?(Class)
+          serialization_map.invert[klass] || raise(UnknownIdentifier, "Could not find mapping for #{klass}")
+        end
+
+        def decode(identifier)
+          serialization_map[identifier] || raise(UnknownIdentifier, "Could not find mapping for #{identifier}")
+        end
+
+        def serialization_map
+          @serialization_map ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/flows/sequence.rb

@@ -10,5 +10,7 @@ module Dynflow
         self << dependent_flow
       end
     end
+
+    Registry.register!(Sequence, 'S')
   end
 end

data/lib/dynflow/persistence.rb

@@ -134,5 +134,13 @@ module Dynflow
         envelope
       end
     end
+
+    def prune_envelopes(receiver_ids)
+      adapter.prune_envelopes(receiver_ids)
+    end
+
+    def prune_undeliverable_envelopes
+      adapter.prune_undeliverable_envelopes
+    end
   end
 end

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -116,6 +116,22 @@ module Dynflow
       def push_envelope(envelope)
         raise NotImplementedError
       end
+
+      def prune_envelopes(receiver_ids)
+        raise NotImplementedError
+      end
+
+      def prune_undeliverable_envelopes
+        raise NotImplementedError
+      end
+
+      def migrate_db
+        raise NotImplementedError
+      end
+
+      def abort_if_pending_migrations!
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -45,13 +45,15 @@ module Dynflow
                                step:    %w(error children) }
 
       def initialize(config)
+        migrate = true
         config = config.dup
         @additional_responsibilities = { coordinator: true, connector: true }
-        if config.is_a?(Hash) && config.key?(:additional_responsibilities)
-          @additional_responsibilities.merge!(config.delete(:additional_responsibilities))
+        if config.is_a?(Hash)
+          @additional_responsibilities.merge!(config.delete(:additional_responsibilities)) if config.key?(:additional_responsibilities)
+          migrate = config.fetch(:migrate, true)
         end
         @db = initialize_db config
-        migrate_db
+        migrate_db if migrate
       end
 
       def transaction(&block)
@@ -198,6 +200,16 @@ module Dynflow
         table(:envelope).insert(prepare_record(:envelope, envelope))
       end
 
+      def prune_envelopes(receiver_ids)
+        connector_feature!
+        table(:envelope).where(receiver_id: receiver_ids).delete
+      end
+
+      def prune_undeliverable_envelopes
+        connector_feature!
+        table(:envelope).where(receiver_id: table(:coordinator_record).select(:id)).invert.delete
+      end
+
       def coordinator_feature!
         unless @additional_responsibilities[:coordinator]
           raise "The sequel persistence adapter coordinator feature used but not enabled in additional_features"
@@ -238,6 +250,14 @@ module Dynflow
           envelopes:            table(:envelope).all.to_a }
       end
 
+      def migrate_db
+        ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
+      end
+
+      def abort_if_pending_migrations!
+        ::Sequel::Migrator.check_current(db, self.class.migrations_path, table: 'dynflow_schema_info')
+      end
+
       private
 
       TABLES = { execution_plan:      :dynflow_execution_plans,
@@ -259,10 +279,6 @@ module Dynflow
         File.expand_path('../sequel_migrations', __FILE__)
       end
 
-      def migrate_db
-        ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
-      end
-
       def prepare_record(table_name, value, base = {}, with_data = true)
         record = base.dup
         if with_data && table(table_name).columns.include?(:data)
@@ -378,7 +394,7 @@ module Dynflow
 
       def dump_data(value)
         return if value.nil?
-        MultiJson.dump Type!(value, Hash, Array)
+        MultiJson.dump Type!(value, Hash, Array, Integer)
       end
 
       def paginate(data_set, options)

data/lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+Sequel.migration do
+  up do
+    alter_table(:dynflow_actions) do
+      drop_index [:execution_plan_uuid, :id]
+    end
+
+    alter_table(:dynflow_execution_plans) do
+      drop_index :uuid
+    end
+
+    alter_table(:dynflow_steps) do
+      drop_index [:execution_plan_uuid, :id]
+    end
+  end
+
+  down do
+    alter_table(:dynflow_actions) do
+      add_index [:execution_plan_uuid, :id], :unique => true
+    end
+
+    alter_table(:dynflow_execution_plans) do
+      add_index :uuid, :unique => true
+    end
+
+    alter_table(:dynflow_steps) do
+      add_index [:execution_plan_uuid, :id], :unique => true
+    end
+  end
+end

data/lib/dynflow/rails.rb

@@ -38,8 +38,8 @@ module Dynflow
       init_world.tap do |world|
         @world = world
         config.run_on_init_hooks(false, world)
+        config.increase_db_pool_size(world)
         unless config.remote?
-          config.increase_db_pool_size(world)
           config.run_on_init_hooks(true, world)
           # leave this just for long-running executors
           unless config.rake_task_with_executor?

data/lib/dynflow/rails/configuration.rb

@@ -96,12 +96,23 @@ module Dynflow
       end
 
       def increase_db_pool_size?
-        !::Rails.env.test? && !remote?
+        !::Rails.env.test? && (!remote? || sidekiq_worker?)
+      end
+
+      def sidekiq_worker?
+        defined?(::Sidekiq) && ::Sidekiq.options[:queues].any?
       end
 
       def calculate_db_pool_size(world)
-        self.db_pool_size || world.config.queues.values.inject(5) do |pool_size, pool_options|
-          pool_size += pool_options[:pool_size]
+        return self.db_pool_size if self.db_pool_size
+
+        base_value = 5
+        if defined?(::Sidekiq)
+          Sidekiq.options[:concurrency] + base_value
+        else
+          world.config.queues.values.inject(base_value) do |pool_size, pool_options|
+            pool_size += pool_options[:pool_size]
+          end
         end
       end
 
@@ -185,8 +196,8 @@ module Dynflow
       end
 
       # Sequel adapter based on Rails app database.yml configuration
-      def initialize_persistence(world)
-        persistence_class.new(default_sequel_adapter_options(world))
+      def initialize_persistence(world, options = {})
+        persistence_class.new(default_sequel_adapter_options(world).merge(options))
       end
     end
   end