dynflow 1.4.7 → 1.6.1

data/lib/dynflow/director.rb

@@ -15,7 +15,8 @@ module Dynflow
               execution_plan_id: String,
               step_id:           Integer,
               event:             Object,
-              result:            Concurrent::Promises::ResolvableFuture
+              result:            Concurrent::Promises::ResolvableFuture,
+              optional:          Algebrick::Types::Boolean
     end
 
     UnprocessableEvent = Class.new(Dynflow::Error)
@@ -52,7 +53,7 @@ module Dynflow
       end
 
       def self.new_from_hash(hash, *_args)
-        self.new(hash[:execution_plan_id], hash[:queue])
+        self.new(hash[:execution_plan_id], hash[:queue], hash[:sender_orchestrator_id])
       end
     end
 
@@ -107,6 +108,26 @@ module Dynflow
       end
     end
 
+    class PlanningWorkItem < WorkItem
+      def execute
+        plan = world.persistence.load_delayed_plan(execution_plan_id)
+        return if plan.nil? || plan.execution_plan.state != :scheduled
+
+        if !plan.start_before.nil? && plan.start_before < Time.now.utc()
+          plan.timeout
+          return
+        end
+
+        world.coordinator.acquire(Coordinator::PlanningLock.new(world, plan.execution_plan_uuid)) do
+          plan.plan
+        end
+        plan.execute
+      rescue => e
+        world.logger.warn e.message
+        world.logger.debug e.backtrace.join("\n")
+      end
+    end
+
     class FinalizeWorkItem < WorkItem
       attr_reader :finalize_steps_data
 
@@ -146,12 +167,18 @@ module Dynflow
       @logger = world.logger
       @execution_plan_managers = {}
       @rescued_steps = {}
+      @planning_plans = []
     end
 
     def current_execution_plan_ids
       @execution_plan_managers.keys
     end
 
+    def handle_planning(execution_plan_uuid)
+      @planning_plans << execution_plan_uuid
+      [PlanningWorkItem.new(execution_plan_uuid, :default, @world.id)]
+    end
+
     def start_execution(execution_plan_id, finished)
       manager = track_execution_plan(execution_plan_id, finished)
       return [] unless manager
@@ -163,6 +190,9 @@ module Dynflow
       execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
       if execution_plan_manager
         execution_plan_manager.event(event)
+      elsif event.optional
+        event.result.reject "no manager for #{event.inspect}"
+        []
       else
         raise Dynflow::Error, "no manager for #{event.inspect}"
       end
@@ -172,9 +202,16 @@ module Dynflow
     end
 
     def work_finished(work)
-      manager = @execution_plan_managers[work.execution_plan_id]
-      return [] unless manager # skip case when getting event from execution plan that is not running anymore
-      unless_done(manager, manager.what_is_next(work))
+      case work
+      when PlanningWorkItem
+        @planning_plans.delete(work.execution_plan_id)
+        @world.persistence.delete_delayed_plans(:execution_plan_uuid => work.execution_plan_id)
+        []
+      else
+        manager = @execution_plan_managers[work.execution_plan_id]
+        return [] unless manager # skip case when getting event from execution plan that is not running anymore
+        unless_done(manager, manager.what_is_next(work))
+      end
     end
 
     # called when there was an unhandled exception during the execution

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -132,11 +132,13 @@ module Dynflow
       end
 
       def dispatch_request(request, client_world_id, request_id)
+        ignore_unknown = false
         executor_id = match request,
-                            (on ~Execution do |execution|
+                            (on ~Execution | ~Planning do |execution|
                                AnyExecutor
                              end),
                             (on ~Event do |event|
+                               ignore_unknown = event.optional
                                find_executor(event.execution_plan_id)
                              end),
                             (on Ping.(~any, ~any) | Status.(~any, ~any) do |receiver_id, _|
@@ -144,7 +146,11 @@ module Dynflow
                              end)
         envelope = Envelope[request_id, client_world_id, executor_id, request]
         if Dispatcher::UnknownWorld === envelope.receiver_id
-          raise Dynflow::Error, "Could not find an executor for #{envelope}"
+          raise Dynflow::Error, "Could not find an executor for #{envelope}" unless ignore_unknown
+
+          message = "Could not find an executor for optional #{envelope}, discarding."
+          log(Logger::DEBUG, message)
+          return respond(envelope, Failed[message])
         end
         connector.send(envelope).value!
       rescue => e

data/lib/dynflow/dispatcher/executor_dispatcher.rb

@@ -9,6 +9,7 @@ module Dynflow
 
       def handle_request(envelope)
         match(envelope.message,
+              on(Planning) { perform_planning(envelope, envelope.message)},
               on(Execution) { perform_execution(envelope, envelope.message) },
               on(Event)     { perform_event(envelope, envelope.message) },
               on(Status)    { get_execution_status(envelope, envelope.message) })
@@ -16,6 +17,13 @@ module Dynflow
 
       protected
 
+      def perform_planning(envelope, planning)
+        @world.executor.plan(planning.execution_plan_id)
+        respond(envelope, Accepted)
+      rescue Dynflow::Error => e
+        respond(envelope, Failed[e.message])
+      end
+
       def perform_execution(envelope, execution)
         allocate_executor(execution.execution_plan_id, envelope.sender_id, envelope.request_id)
         execution_lock = Coordinator::ExecutionLock.new(@world, execution.execution_plan_id, envelope.sender_id, envelope.request_id)
@@ -52,12 +60,14 @@ module Dynflow
           end
         end
         if event_request.time.nil? || event_request.time < Time.now
-          @world.executor.event(envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, future)
+          @world.executor.event(envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, future,
+                                optional: event_request.optional)
         else
           @world.clock.ping(
             @world.executor,
             event_request.time,
-            Director::Event[envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, Concurrent::Promises.resolvable_future],
+            Director::Event[envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, Concurrent::Promises.resolvable_future,
+                            event_request.optional],
             :delayed_event
           )
           # resolves the future right away - currently we do not wait for the clock ping

data/lib/dynflow/dispatcher.rb

@@ -6,13 +6,18 @@ module Dynflow
         fields! execution_plan_id: String,
                 step_id:           Integer,
                 event:             Object,
-                time:              type { variants Time, NilClass }
+                time:              type { variants Time, NilClass },
+                optional:          Algebrick::Types::Boolean
       end
 
       Execution = type do
         fields! execution_plan_id: String
       end
 
+      Planning = type do
+        fields! execution_plan_id: String
+      end
+
       Ping = type do
         fields! receiver_id: String,
                 use_cache: type { variants TrueClass, FalseClass }
@@ -23,7 +28,7 @@ module Dynflow
                 execution_plan_id: type { variants String, NilClass }
       end
 
-      variants Event, Execution, Ping, Status
+      variants Event, Execution, Ping, Status, Planning
     end
 
     Response = Algebrick.type do

data/lib/dynflow/execution_history.rb

@@ -12,7 +12,7 @@ module Dynflow
 
     module Event
       def inspect
-        "#{Time.at(time).utc}: #{name}".tap { |s| s << " @ #{world_id}" if world_id }
+        ["#{Time.at(time).utc}: #{name}", world_id].compact.join(' @ ')
       end
     end
 

data/lib/dynflow/execution_plan/hooks.rb

@@ -21,7 +21,7 @@ module Dynflow
         # @param class_name [Class] class of the hook to be run
         # @param on [Symbol, Array<Symbol>] when should the hook be run, one of {HOOK_KINDS}
         # @return [void]
-        def use(class_name, on: HOOK_KINDS)
+        def use(class_name, on: ExecutionPlan.states)
           on = Array[on] unless on.kind_of?(Array)
           validate_kinds!(on)
           if hooks[class_name]

data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb

@@ -31,6 +31,7 @@ module Dynflow
         action = persistence.load_action(self)
         yield action
         persistence.save_action(execution_plan_id, action)
+        persistence.save_output_chunks(execution_plan_id, action.id, action.pending_output_chunks)
         save
 
         return self

data/lib/dynflow/execution_plan.rb

@@ -254,6 +254,7 @@ module Dynflow
     def delay(caller_action, action_class, delay_options, *args)
       save
       @root_plan_step = add_scheduling_step(action_class, caller_action)
+      run_hooks(:pending)
       serializer = root_plan_step.delay(delay_options, args)
       delayed_plan = DelayedPlan.new(@world,
                                      id,
@@ -276,7 +277,9 @@ module Dynflow
       raise "Unexpected options #{options.keys.inspect}" unless options.empty?
       save
       @root_plan_step = add_plan_step(action_class, caller_action)
-      @root_plan_step.save
+      step = @root_plan_step.save
+      run_hooks(:pending)
+      step
     end
 
     def plan(*args)
@@ -418,6 +421,14 @@ module Dynflow
       end
     end
 
+    def self.load_flow(flow_hash)
+      if flow_hash.is_a? Hash
+        Flows::Abstract.from_hash(flow_hash)
+      else
+        Flows::Abstract.decode(flow_hash)
+      end
+    end
+
     def to_hash
       recursive_to_hash id:                id,
                         class:             self.class.to_s,
@@ -425,8 +436,8 @@ module Dynflow
                         state:             state,
                         result:            result,
                         root_plan_step_id: root_plan_step && root_plan_step.id,
-                        run_flow:          run_flow,
-                        finalize_flow:     finalize_flow,
+                        run_flow:          run_flow.encode,
+                        finalize_flow:     finalize_flow.encode,
                         step_ids:          steps.map { |id, _| id },
                         started_at:        time_to_str(started_at),
                         ended_at:          time_to_str(ended_at),
@@ -448,8 +459,8 @@ module Dynflow
                hash[:label],
                hash[:state],
                steps[hash[:root_plan_step_id]],
-               Flows::Abstract.from_hash(hash[:run_flow]),
-               Flows::Abstract.from_hash(hash[:finalize_flow]),
+               load_flow(hash[:run_flow]),
+               load_flow(hash[:finalize_flow]),
                steps,
                string_to_time(hash[:started_at]),
                string_to_time(hash[:ended_at]),

data/lib/dynflow/executors/abstract/core.rb

@@ -35,9 +35,18 @@ module Dynflow
           handle_work(@director.handle_event(event))
         end
 
+        def handle_planning(execution_plan_id)
+          if terminating?
+            raise Dynflow::Error,
+                  "cannot accept event: #{event} core is terminating"
+          end
+
+          handle_work(@director.handle_planning(execution_plan_id))
+        end
+
         def plan_events(delayed_events)
           delayed_events.each do |event|
-            @world.plan_event(event.execution_plan_id, event.step_id, event.event, event.time)
+            @world.plan_event(event.execution_plan_id, event.step_id, event.event, event.time, optional: event.optional)
           end
         end
 

data/lib/dynflow/executors/parallel.rb

@@ -33,11 +33,15 @@ module Dynflow
         raise e
       end
 
-      def event(request_id, execution_plan_id, step_id, event, future = nil)
-        @core.ask([:handle_event, Director::Event[request_id, execution_plan_id, step_id, event, future]])
+      def event(request_id, execution_plan_id, step_id, event, future = nil, optional: false)
+        @core.ask([:handle_event, Director::Event[request_id, execution_plan_id, step_id, event, future, optional]])
         future
       end
 
+      def plan(execution_plan_id)
+        @core.ask([:handle_planning, execution_plan_id])
+      end
+
       def delayed_event(director_event)
         @core.ask([:handle_event, director_event])
         director_event.result

data/lib/dynflow/extensions/msgpack.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+require 'msgpack'
+
+module Dynflow
+  module Extensions
+    module MsgPack
+      module Time
+        def to_msgpack(out = ''.dup)
+          ::MessagePack.pack(self, out)
+          out
+        end
+      end
+
+      ::Time.include ::Dynflow::Extensions::MsgPack::Time
+      ::MessagePack::DefaultFactory.register_type(0x00, Time, packer: MessagePack::Time::Packer, unpacker: MessagePack::Time::Unpacker)
+
+      begin
+        require 'active_support/time_with_zone'
+        unpacker = ->(payload) do
+          tv = MessagePack::Timestamp.from_msgpack_ext(payload)
+          ::Time.zone.at(tv.sec, tv.nsec, :nanosecond)
+        end
+        ::ActiveSupport::TimeWithZone.include ::Dynflow::Extensions::MsgPack::Time
+        ::MessagePack::DefaultFactory.register_type(0x01, ActiveSupport::TimeWithZone, packer: MessagePack::Time::Packer, unpacker: unpacker)
+
+        ::DateTime.include ::Dynflow::Extensions::MsgPack::Time
+        ::MessagePack::DefaultFactory.register_type(0x02, DateTime,
+                                                    packer: ->(datetime) { MessagePack::Time::Packer.(datetime.to_time) },
+                                                    unpacker: ->(payload) { unpacker.(payload).to_datetime })
+
+        ::Date.include ::Dynflow::Extensions::MsgPack::Time
+        ::MessagePack::DefaultFactory.register_type(0x03, Date,
+                                                    packer: ->(date) { MessagePack::Time::Packer.(date.to_time) },
+                                                    unpacker: ->(payload) { unpacker.(payload).to_date })
+      rescue LoadError
+        # This is fine
+        nil
+      end
+    end
+  end
+end

data/lib/dynflow/extensions.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Dynflow
+  module Extensions
+    require 'dynflow/extensions/msgpack'
+  end
+end

data/lib/dynflow/flows/abstract.rb

@@ -32,6 +32,20 @@ module Dynflow
       def flatten!
         raise NotImplementedError
       end
+
+      def self.new_from_hash(hash)
+        check_class_matching hash
+        new(hash[:flows].map { |flow_hash| from_hash(flow_hash) })
+      end
+
+      def self.decode(data)
+        if data.is_a? Integer
+          Flows::Atom.new(data)
+        else
+          kind, *subflows = data
+          Registry.decode(kind).new(subflows.map { |subflow| self.decode(subflow) })
+        end
+      end
     end
   end
 end

data/lib/dynflow/flows/abstract_composed.rb

@@ -11,8 +11,8 @@ module Dynflow
         @flows = flows
       end
 
-      def to_hash
-        super.merge recursive_to_hash(:flows => flows)
+      def encode
+        [Registry.encode(self)] + flows.map(&:encode)
       end
 
       def <<(v)
@@ -61,11 +61,6 @@ module Dynflow
 
       protected
 
-      def self.new_from_hash(hash)
-        check_class_matching hash
-        new(hash[:flows].map { |flow_hash| from_hash(flow_hash) })
-      end
-
       # adds the +new_flow+ in a way that it's in sequence with
       # the +satisfying_flows+
       def add_to_sequence(satisfying_flows, new_flow)

data/lib/dynflow/flows/atom.rb

@@ -5,8 +5,8 @@ module Dynflow
 
       attr_reader :step_id
 
-      def to_hash
-        super.merge(:step_id => step_id)
+      def encode
+        step_id
       end
 
       def initialize(step_id)

data/lib/dynflow/flows/concurrence.rb

@@ -25,5 +25,7 @@ module Dynflow
         return Concurrence.new(extracted_sub_flows)
       end
     end
+
+    Registry.register!(Concurrence, 'C')
   end
 end

data/lib/dynflow/flows/registry.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+module Dynflow
+  module Flows
+    class Registry
+      class IdentifierTaken < ArgumentError; end
+      class UnknownIdentifier < ArgumentError; end
+
+      class << self
+        def register!(klass, identifier)
+          if (found = serialization_map[identifier])
+            raise IdentifierTaken, "Error setting up mapping #{identifier} to #{klass}, it already maps to #{found}"
+          else
+            serialization_map.update(identifier => klass)
+          end
+        end
+
+        def encode(klass)
+          klass = klass.class unless klass.is_a?(Class)
+          serialization_map.invert[klass] || raise(UnknownIdentifier, "Could not find mapping for #{klass}")
+        end
+
+        def decode(identifier)
+          serialization_map[identifier] || raise(UnknownIdentifier, "Could not find mapping for #{identifier}")
+        end
+
+        def serialization_map
+          @serialization_map ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/flows/sequence.rb

@@ -10,5 +10,7 @@ module Dynflow
         self << dependent_flow
       end
     end
+
+    Registry.register!(Sequence, 'S')
   end
 end

data/lib/dynflow/flows.rb

@@ -4,6 +4,7 @@ require 'forwardable'
 module Dynflow
   module Flows
 
+    require 'dynflow/flows/registry'
     require 'dynflow/flows/abstract'
     require 'dynflow/flows/atom'
     require 'dynflow/flows/abstract_composed'

data/lib/dynflow/persistence.rb

@@ -46,6 +46,16 @@ module Dynflow
       adapter.save_action(execution_plan_id, action.id, action.to_hash)
     end
 
+    def save_output_chunks(execution_plan_id, action_id, chunks)
+      return if chunks.empty?
+
+      adapter.save_output_chunks(execution_plan_id, action_id, chunks)
+    end
+
+    def load_output_chunks(execution_plan_id, action_id)
+      adapter.load_output_chunks(execution_plan_id, action_id)
+    end
+
     def find_execution_plans(options)
       adapter.find_execution_plans(options).map do |execution_plan_hash|
         ExecutionPlan.new_from_hash(execution_plan_hash, @world)

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 require 'sequel'
-require 'multi_json'
+require 'msgpack'
 require 'fileutils'
 require 'csv'
 
+# rubocop:disable Metrics/ClassLength
 module Dynflow
   module PersistenceAdapters
 
@@ -37,12 +38,14 @@ module Dynflow
                                             class action_class execution_plan_uuid queue),
                     envelope:            %w(receiver_id),
                     coordinator_record:  %w(id owner_id class),
-                    delayed:             %w(execution_plan_uuid start_at start_before args_serializer frozen)}
+                    delayed:             %w(execution_plan_uuid start_at start_before args_serializer frozen),
+                    output_chunk:        %w(execution_plan_uuid action_id kind timestamp) }
 
       SERIALIZABLE_COLUMNS = { action:  %w(input output),
                                delayed: %w(serialized_args),
                                execution_plan: %w(run_flow finalize_flow execution_history step_ids),
-                               step:    %w(error children) }
+                               step:    %w(error children),
+                               output_chunk: %w(chunk) }
 
       def initialize(config)
         migrate = true
@@ -83,15 +86,17 @@ module Dynflow
             table(:delayed).where(execution_plan_uuid: uuids).delete
 
             steps = table(:step).where(execution_plan_uuid: uuids)
-            backup_to_csv(steps, backup_dir, 'steps.csv') if backup_dir
+            backup_to_csv(:step, steps, backup_dir, 'steps.csv') if backup_dir
             steps.delete
 
+            output_chunks = table(:output_chunk).where(execution_plan_uuid: uuids).delete
+
             actions = table(:action).where(execution_plan_uuid: uuids)
-            backup_to_csv(actions, backup_dir, 'actions.csv') if backup_dir
+            backup_to_csv(:action, actions, backup_dir, 'actions.csv') if backup_dir
             actions.delete
 
             execution_plans = table(:execution_plan).where(uuid: uuids)
-            backup_to_csv(execution_plans, backup_dir, 'execution_plans.csv') if backup_dir
+            backup_to_csv(:execution_plan, execution_plans, backup_dir, 'execution_plans.csv') if backup_dir
             count += execution_plans.delete
           end
         end
@@ -173,6 +178,18 @@ module Dynflow
         save :action, { execution_plan_uuid: execution_plan_id, id: action_id }, value, with_data: false
       end
 
+      def save_output_chunks(execution_plan_id, action_id, chunks)
+        chunks.each do |chunk|
+          chunk[:execution_plan_uuid] = execution_plan_id
+          chunk[:action_id] = action_id
+          save :output_chunk, {}, chunk, with_data: false
+        end
+      end
+
+      def load_output_chunks(execution_plan_id, action_id)
+        load_records :output_chunk, { execution_plan_uuid: execution_plan_id, action_id: action_id }, [:timestamp, :kind, :chunk]
+      end
+
       def connector_feature!
         unless @additional_responsibilities[:connector]
           raise "The sequel persistence adapter connector feature used but not enabled in additional_features"
@@ -265,14 +282,16 @@ module Dynflow
                  step:                :dynflow_steps,
                  envelope:            :dynflow_envelopes,
                  coordinator_record:  :dynflow_coordinator_records,
-                 delayed:             :dynflow_delayed_plans }
+                 delayed:             :dynflow_delayed_plans,
+                 output_chunk:        :dynflow_output_chunks }
 
       def table(which)
         db[TABLES.fetch(which)]
       end
 
       def initialize_db(db_path)
-        ::Sequel.connect db_path
+        logger = Logger.new($stderr) if ENV['DYNFLOW_SQL_LOG']
+        ::Sequel.connect db_path, logger: logger
       end
 
       def self.migrations_path
@@ -281,10 +300,15 @@ module Dynflow
 
       def prepare_record(table_name, value, base = {}, with_data = true)
         record = base.dup
-        if with_data && table(table_name).columns.include?(:data)
+        has_data_column = table(table_name).columns.include?(:data)
+        if with_data && has_data_column
           record[:data] = dump_data(value)
         else
-          record[:data] = nil
+          if has_data_column
+            record[:data] = nil
+          else
+            record.delete(:data)
+          end
           record.merge! serialize_columns(table_name, value)
         end
 
@@ -339,7 +363,11 @@ module Dynflow
         records = with_retry do
           filtered = table.filter(Utils.symbolize_keys(condition))
           # Filter out requested columns which the table doesn't have, load data just in case
-          filtered = filtered.select(:data, *(table.columns & keys)) unless keys.nil?
+          unless keys.nil?
+            columns = table.columns & keys
+            columns |= [:data] if table.columns.include?(:data)
+            filtered = filtered.select(*columns)
+          end
           filtered.all
         end
         records = records.map { |record| load_data(record, what) }
@@ -355,11 +383,11 @@ module Dynflow
         hash = if record[:data].nil?
                  SERIALIZABLE_COLUMNS.fetch(what, []).each do |key|
                    key = key.to_sym
-                   record[key] = MultiJson.load(record[key]) unless record[key].nil?
+                   record[key] = MessagePack.unpack((record[key])) unless record[key].nil?
                  end
                  record
                else
-                 MultiJson.load(record[:data])
+                 MessagePack.unpack(record[:data])
                end
         Utils.indifferent_hash(hash)
       end
@@ -368,7 +396,7 @@ module Dynflow
         FileUtils.mkdir_p(backup_dir) unless File.directory?(backup_dir)
       end
 
-      def backup_to_csv(dataset, backup_dir, file_name)
+      def backup_to_csv(table_name, dataset, backup_dir, file_name)
         ensure_backup_dir(backup_dir)
         csv_file = File.join(backup_dir, file_name)
         appending = File.exist?(csv_file)
@@ -376,7 +404,12 @@ module Dynflow
         File.open(csv_file, 'a') do |csv|
           csv << columns.to_csv unless appending
           dataset.each do |row|
-            csv << columns.collect { |col| row[col] }.to_csv
+            values = columns.map do |col|
+              value = row[col]
+              value = value.unpack('H*').first if value && SERIALIZABLE_COLUMNS.fetch(table_name, []).include?(col.to_s)
+              value
+            end
+            csv << values.to_csv
           end
         end
         dataset
@@ -394,7 +427,8 @@ module Dynflow
 
       def dump_data(value)
         return if value.nil?
-        MultiJson.dump Type!(value, Hash, Array)
+        packed = MessagePack.pack(Type!(value, Hash, Array, Integer, String))
+        ::Sequel.blob(packed)
       end
 
       def paginate(data_set, options)
@@ -477,3 +511,4 @@ module Dynflow
     end
   end
 end
+# rubocop:enable Metrics/ClassLength