dynflow 1.5.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b33bcad7864102ad9f0f3bb654c7990b3037b7b0620b770d5f34f1495402855
-  data.tar.gz: f4089f0cb793384a764f4aa042268c19d174c989c5d348f63e126eac5fb94716
+  metadata.gz: f660ffddfd3e3c7ea4b8414d2e6f27f48d897d34280ff3bec3c8ab6e360b0bfe
+  data.tar.gz: 46cd643c84f09640a07f00eafc3f4d3feda65746d3c353338d8141277fd77e03
 SHA512:
-  metadata.gz: c119f0ce0d3605012206b083b829bc85194d676f762d61a394be6257d376c56388efd72f4a77f78445ece78942c5eb561728031e9a703767113eab7780b9d0e3
-  data.tar.gz: 136dafb81d0766e9c1041a9a52896b4cbe0d300482be729900fd9fe8f83e6095bbc084a6d43d0aa9012b436544690d3c2d774d46f0b9c573775b3707daec3428
+  metadata.gz: 99e59816fbf809403a87bbb1f96ff39958d3d059572de7a66f6205df7c48867baa50d77636d287aebb8c69d73db2cef686860520c6e7db27dbfa8217f5570715
+  data.tar.gz: 315b37f302e1c29eae52d00acabd82b605d88b7982ba3e01c920dd18619967115875ec5a1facffc6299280f01aaa61db8162ce3e02bbbd0ba06a95add1b88d64

data/dynflow.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.3.0'
 
   s.add_dependency "multi_json"
+  s.add_dependency "msgpack", '~> 1.3.3'
   s.add_dependency "apipie-params"
   s.add_dependency "algebrick", '~> 0.7.0'
   s.add_dependency "concurrent-ruby", '~> 1.1.3'

data/examples/chunked_output_benchmark.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative 'example_helper'
+require 'benchmark'
+
+WORDS = File.readlines('/usr/share/dict/words').map(&:chomp).freeze
+COUNT = WORDS.count
+
+module Common
+  def main_loop
+    if output[:current] < input[:limit]
+      consumed = yield
+      output[:current] += consumed
+      plan_event(nil)
+      suspend
+    end
+  end
+
+  def batch
+    WORDS.drop(output[:current]).take(input[:chunk])
+  end
+end
+
+class Regular < ::Dynflow::Action
+  include Common
+
+  def run(event = nil)
+    output[:current] ||= 0
+    output[:words] ||= []
+
+    main_loop do
+      words = batch
+      output[:words] << words
+      words.count
+    end
+  end
+end
+
+class Chunked < ::Dynflow::Action
+  include Common
+
+  def run(event = nil)
+    output[:current] ||= 0
+
+    main_loop do
+      words = batch
+      output_chunk(words)
+      words.count
+    end
+  end
+end
+
+if $0 == __FILE__
+  ExampleHelper.world.action_logger.level = 4
+  ExampleHelper.world.logger.level = 4
+
+  Benchmark.bm do |bm|
+    bm.report('regular    1000 by    100') { ExampleHelper.world.trigger(Regular, limit: 1000, chunk: 100).finished.wait }
+    bm.report('chunked    1000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 1000, chunk: 100).finished.wait }
+
+    bm.report('regular  10_000 by    100') { ExampleHelper.world.trigger(Regular, limit: 10_000, chunk: 100).finished.wait }
+    bm.report('chunked  10_000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 10_000, chunk: 100).finished.wait }
+
+    bm.report('regular  10_000 by   1000') { ExampleHelper.world.trigger(Regular, limit: 10_000, chunk: 1000).finished.wait }
+    bm.report('chunked  10_000 by   1000') { ExampleHelper.world.trigger(Chunked, limit: 10_000, chunk: 1000).finished.wait }
+
+    bm.report('regular 100_000 by    100') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 100).finished.wait }
+    bm.report('chunked 100_000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 100).finished.wait }
+
+    bm.report('regular 100_000 by   1000') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 1000).finished.wait }
+    bm.report('chunked 100_000 by   1000') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 1000).finished.wait }
+
+    bm.report('regular 100_000 by 10_000') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 10_000).finished.wait }
+    bm.report('chunked 100_000 by 10_000') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 10_000).finished.wait }
+  end
+end

data/extras/expand/main.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/hex"
+	"encoding/json"
+	"github.com/vmihailenco/msgpack"
+	"io"
+	"os"
+)
+
+// dynflow_steps
+// 0                   1  2         3    4     5          6        7         8              9             10              11    12    13           14       15
+// execution_plan_uuid,id,action_id,data,state,started_at,ended_at,real_time,execution_time,progress_done,progress_weight,class,error,action_class,children,queue
+//
+// encoded columns are:
+// 3 - data
+// 12 - error
+// 14 - children
+
+// dynflow_actions
+// 0                   1  2    3                        4                5     6     7      8            9           10
+// execution_plan_uuid,id,data,caller_execution_plan_id,caller_action_id,class,input,output,plan_step_id,run_step_id,finalize_step_id
+//
+// encoded columns are:
+// 2 - data
+// 6 - input
+// 7 - output
+
+// dynflow_execution_plans
+// Without msgpack
+// 0    1    2     3      4          5        6         7              8     9     10       11            12                13                14
+// uuid,data,state,result,started_at,ended_at,real_time,execution_time,label,class,run_flow,finalize_flow,execution_history,root_plan_step_id,step_ids
+
+// With msgpack
+// 0    1    2     3      4          5        6         7              8     9     10                11       12            13                14
+// uuid,data,state,result,started_at,ended_at,real_time,execution_time,label,class,root_plan_step_id,run_flow,finalize_flow,execution_history,step_ids
+//
+// 1 - data
+// 11 - run_flow
+// 12 - finalize_flow
+// 13 - execution_history
+// 14 - step_ids
+
+func main() {
+	reader := csv.NewReader(os.Stdin)
+	writer := csv.NewWriter(os.Stdout)
+	defer writer.Flush()
+
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+
+		writer.Write(processRow(record))
+	}
+}
+
+func processRow(record []string) []string {
+	// Execution plan exports have 15 fields, other exports have different counts
+	if len(record) == 15 {
+		record = expandExecutionPlan(record)
+	}
+
+	for i, r := range record {
+		record[i] = reencodeField(r)
+	}
+
+	return record
+}
+
+func expandExecutionPlan(record []string) []string {
+	var flow_columns [2]int
+
+	// The step_ids field should be a safe indicator
+	if isHexEncoded(record[14]) {
+		flow_columns = [...]int{11, 12}
+	} else {
+		flow_columns = [...]int{10, 11}
+	}
+
+	for _, i := range flow_columns {
+		record[i] = expandFlow(record[i])
+	}
+	return record
+}
+
+func isHexEncoded(field string) bool {
+	return len(field) >= 2 && field[0:2] == "\\x"
+}
+
+func reencodeField(field string) string {
+	decoded, err := decode(field)
+	if err != nil {
+		return field
+	}
+
+	return encode(decoded)
+}
+
+func decode(field string) (interface{}, error) {
+	var intermediate interface{}
+	bytes := []byte(field)
+
+	if isHexEncoded(field) {
+		decoded_bytes, err := hex.DecodeString(field[2:])
+		if err != nil {
+			return "", err
+		}
+
+		err = msgpack.Unmarshal(decoded_bytes, &intermediate)
+		if err != nil {
+			return "", err
+		}
+
+		return intermediate, nil
+	}
+
+	err := json.Unmarshal(bytes, &intermediate)
+	if err != nil {
+		return "", err
+	}
+
+	return intermediate, nil
+}
+
+func encode(data interface{}) string {
+	result, err := json.Marshal(data)
+	if err != nil {
+		panic(err)
+	}
+
+	return string(result)
+}
+
+func expandFlow(field string) string {
+	intermediate, err := decode(field)
+	if err != nil {
+		return field
+	}
+
+	var result map[string]interface{}
+	switch intermediate.(type) {
+	// old style hash
+	case map[string]interface{}:
+		result = intermediate.(map[string]interface{})
+	// newer compact S-expression like representation
+	case []interface{}, float64:
+		result = expandCompactFlow(intermediate)
+	}
+
+	return encode(result)
+}
+
+func expandCompactFlow(flow interface{}) map[string]interface{} {
+	result := make(map[string]interface{})
+	switch flow.(type) {
+	case []interface{}:
+		switch flow.([]interface{})[0] {
+		case "S":
+			result["class"] = "Dynflow::Flows::Sequence"
+		case "C":
+			result["class"] = "Dynflow::Flows::Concurrence"
+		default:
+			panic("Unknown flow type")
+		}
+		var subflows []interface{}
+		for subflow := range flow.([]interface{})[1:] {
+			subflows = append(subflows, expandCompactFlow(subflow))
+		}
+		result["flows"] = subflows
+	case float64, int:
+		result["class"] = "Dynflow::Flows::Atom"
+		result["step_id"] = flow
+	default:
+		panic("Unknown flow type")
+	}
+	return result
+}

data/lib/dynflow/action.rb

@@ -105,7 +105,8 @@ module Dynflow
 
     attr_reader :world, :phase, :execution_plan_id, :id, :input,
                 :plan_step_id, :run_step_id, :finalize_step_id,
-                :caller_execution_plan_id, :caller_action_id
+                :caller_execution_plan_id, :caller_action_id,
+                :pending_output_chunks
 
     middleware.use Action::Progress::Calculate
 
@@ -133,6 +134,7 @@ module Dynflow
 
       @input  = OutputReference.deserialize getter.(:input, phase?(Run, Finalize, Present))
       @output = OutputReference.deserialize getter.(:output, false) if phase? Run, Finalize, Present
+      @pending_output_chunks = [] if phase? Run, Finalize
     end
 
     def phase?(*phases)
@@ -169,6 +171,14 @@ module Dynflow
       end
     end
 
+    def output_chunk(chunk, kind: nil, timestamp: Time.now)
+      @pending_output_chunks << { chunk: chunk, kind: kind, timestamp: timestamp }
+    end
+
+    def stored_output_chunks
+      @output_chunks ||= world.persistence.load_output_chunks(@execution_plan_id, @id)
+    end
+
     def caller_action
       phase! Present
       return nil if @caller_action_id

data/lib/dynflow/delayed_executors/abstract_core.rb

@@ -46,24 +46,21 @@ module Dynflow
 
       def process(delayed_plans, check_time)
         processed_plan_uuids = []
+        dispatched_plan_uuids = []
+        planning_locks = world.coordinator.find_records(class: Coordinator::PlanningLock.name)
         delayed_plans.each do |plan|
-          next if plan.frozen
+          next if plan.frozen || locked_for_planning?(planning_locks, plan)
           fix_plan_state(plan)
           with_error_handling do
             if plan.execution_plan.state != :scheduled
               # in case the previous process was terminated after running the plan, but before deleting the delayed plan record.
               @logger.info("Execution plan #{plan.execution_plan_uuid} is expected to be in 'scheduled' state, was '#{plan.execution_plan.state}', skipping")
-            elsif !plan.start_before.nil? && plan.start_before < check_time
-              @logger.debug "Failing plan #{plan.execution_plan_uuid}"
-              plan.timeout
+              processed_plan_uuids << plan.execution_plan_uuid
             else
               @logger.debug "Executing plan #{plan.execution_plan_uuid}"
-              Executors.run_user_code do
-                plan.plan
-                plan.execute
-              end
+              world.plan_request(plan.execution_plan_uuid)
+              dispatched_plan_uuids << plan.execution_plan_uuid
             end
-            processed_plan_uuids << plan.execution_plan_uuid
           end
         end
         world.persistence.delete_delayed_plans(:execution_plan_uuid => processed_plan_uuids) unless processed_plan_uuids.empty?
@@ -72,6 +69,7 @@ module Dynflow
       private
 
       # handle the case, where the process was termintated while planning was in progress before
+      # TODO: Doing execution plan updates in orchestrator is bad
       def fix_plan_state(plan)
         if plan.execution_plan.state == :planning
           @logger.info("Execution plan #{plan.execution_plan_uuid} is expected to be in 'scheduled' state, was '#{plan.execution_plan.state}', auto-fixing")
@@ -79,6 +77,10 @@ module Dynflow
           plan.execution_plan.save
         end
       end
+
+      def locked_for_planning?(planning_locks, plan)
+        planning_locks.any? { |lock| lock.execution_plan_id == plan.execution_plan_uuid }
+      end
     end
   end
 end

data/lib/dynflow/director.rb

@@ -53,7 +53,7 @@ module Dynflow
       end
 
       def self.new_from_hash(hash, *_args)
-        self.new(hash[:execution_plan_id], hash[:queue])
+        self.new(hash[:execution_plan_id], hash[:queue], hash[:sender_orchestrator_id])
       end
     end
 
@@ -108,6 +108,26 @@ module Dynflow
       end
     end
 
+    class PlanningWorkItem < WorkItem
+      def execute
+        plan = world.persistence.load_delayed_plan(execution_plan_id)
+        return if plan.nil? || plan.execution_plan.state != :scheduled
+
+        if !plan.start_before.nil? && plan.start_before < Time.now.utc()
+          plan.timeout
+          return
+        end
+
+        world.coordinator.acquire(Coordinator::PlanningLock.new(world, plan.execution_plan_uuid)) do
+          plan.plan
+        end
+        plan.execute
+      rescue => e
+        world.logger.warn e.message
+        world.logger.debug e.backtrace.join("\n")
+      end
+    end
+
     class FinalizeWorkItem < WorkItem
       attr_reader :finalize_steps_data
 
@@ -147,12 +167,18 @@ module Dynflow
       @logger = world.logger
       @execution_plan_managers = {}
       @rescued_steps = {}
+      @planning_plans = []
     end
 
     def current_execution_plan_ids
       @execution_plan_managers.keys
     end
 
+    def handle_planning(execution_plan_uuid)
+      @planning_plans << execution_plan_uuid
+      [PlanningWorkItem.new(execution_plan_uuid, :default, @world.id)]
+    end
+
     def start_execution(execution_plan_id, finished)
       manager = track_execution_plan(execution_plan_id, finished)
       return [] unless manager
@@ -176,9 +202,16 @@ module Dynflow
     end
 
     def work_finished(work)
-      manager = @execution_plan_managers[work.execution_plan_id]
-      return [] unless manager # skip case when getting event from execution plan that is not running anymore
-      unless_done(manager, manager.what_is_next(work))
+      case work
+      when PlanningWorkItem
+        @planning_plans.delete(work.execution_plan_id)
+        @world.persistence.delete_delayed_plans(:execution_plan_uuid => work.execution_plan_id)
+        []
+      else
+        manager = @execution_plan_managers[work.execution_plan_id]
+        return [] unless manager # skip case when getting event from execution plan that is not running anymore
+        unless_done(manager, manager.what_is_next(work))
+      end
     end
 
     # called when there was an unhandled exception during the execution

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -134,7 +134,7 @@ module Dynflow
       def dispatch_request(request, client_world_id, request_id)
         ignore_unknown = false
         executor_id = match request,
-                            (on ~Execution do |execution|
+                            (on ~Execution | ~Planning do |execution|
                                AnyExecutor
                              end),
                             (on ~Event do |event|

data/lib/dynflow/dispatcher/executor_dispatcher.rb

@@ -9,6 +9,7 @@ module Dynflow
 
       def handle_request(envelope)
         match(envelope.message,
+              on(Planning) { perform_planning(envelope, envelope.message)},
               on(Execution) { perform_execution(envelope, envelope.message) },
               on(Event)     { perform_event(envelope, envelope.message) },
               on(Status)    { get_execution_status(envelope, envelope.message) })
@@ -16,6 +17,13 @@ module Dynflow
 
       protected
 
+      def perform_planning(envelope, planning)
+        @world.executor.plan(planning.execution_plan_id)
+        respond(envelope, Accepted)
+      rescue Dynflow::Error => e
+        respond(envelope, Failed[e.message])
+      end
+
       def perform_execution(envelope, execution)
         allocate_executor(execution.execution_plan_id, envelope.sender_id, envelope.request_id)
         execution_lock = Coordinator::ExecutionLock.new(@world, execution.execution_plan_id, envelope.sender_id, envelope.request_id)

data/lib/dynflow/dispatcher.rb

@@ -14,6 +14,10 @@ module Dynflow
         fields! execution_plan_id: String
       end
 
+      Planning = type do
+        fields! execution_plan_id: String
+      end
+
       Ping = type do
         fields! receiver_id: String,
                 use_cache: type { variants TrueClass, FalseClass }
@@ -24,7 +28,7 @@ module Dynflow
                 execution_plan_id: type { variants String, NilClass }
       end
 
-      variants Event, Execution, Ping, Status
+      variants Event, Execution, Ping, Status, Planning
     end
 
     Response = Algebrick.type do

data/lib/dynflow/execution_plan/hooks.rb

@@ -21,7 +21,7 @@ module Dynflow
         # @param class_name [Class] class of the hook to be run
         # @param on [Symbol, Array<Symbol>] when should the hook be run, one of {HOOK_KINDS}
         # @return [void]
-        def use(class_name, on: HOOK_KINDS)
+        def use(class_name, on: ExecutionPlan.states)
           on = Array[on] unless on.kind_of?(Array)
           validate_kinds!(on)
           if hooks[class_name]

data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb

@@ -31,6 +31,7 @@ module Dynflow
         action = persistence.load_action(self)
         yield action
         persistence.save_action(execution_plan_id, action)
+        persistence.save_output_chunks(execution_plan_id, action.id, action.pending_output_chunks)
         save
 
         return self

data/lib/dynflow/execution_plan.rb

@@ -254,6 +254,7 @@ module Dynflow
     def delay(caller_action, action_class, delay_options, *args)
       save
       @root_plan_step = add_scheduling_step(action_class, caller_action)
+      run_hooks(:pending)
       serializer = root_plan_step.delay(delay_options, args)
       delayed_plan = DelayedPlan.new(@world,
                                      id,
@@ -276,7 +277,9 @@ module Dynflow
       raise "Unexpected options #{options.keys.inspect}" unless options.empty?
       save
       @root_plan_step = add_plan_step(action_class, caller_action)
-      @root_plan_step.save
+      step = @root_plan_step.save
+      run_hooks(:pending)
+      step
     end
 
     def plan(*args)

data/lib/dynflow/executors/abstract/core.rb

@@ -35,6 +35,15 @@ module Dynflow
           handle_work(@director.handle_event(event))
         end
 
+        def handle_planning(execution_plan_id)
+          if terminating?
+            raise Dynflow::Error,
+                  "cannot accept event: #{event} core is terminating"
+          end
+
+          handle_work(@director.handle_planning(execution_plan_id))
+        end
+
         def plan_events(delayed_events)
           delayed_events.each do |event|
             @world.plan_event(event.execution_plan_id, event.step_id, event.event, event.time, optional: event.optional)

data/lib/dynflow/executors/parallel.rb

@@ -38,6 +38,10 @@ module Dynflow
         future
       end
 
+      def plan(execution_plan_id)
+        @core.ask([:handle_planning, execution_plan_id])
+      end
+
       def delayed_event(director_event)
         @core.ask([:handle_event, director_event])
         director_event.result

data/lib/dynflow/extensions/msgpack.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+require 'msgpack'
+
+module Dynflow
+  module Extensions
+    module MsgPack
+      module Time
+        def to_msgpack(out = ''.dup)
+          ::MessagePack.pack(self, out)
+          out
+        end
+      end
+
+      ::Time.include ::Dynflow::Extensions::MsgPack::Time
+      ::MessagePack::DefaultFactory.register_type(0x00, Time, packer: MessagePack::Time::Packer, unpacker: MessagePack::Time::Unpacker)
+
+      begin
+        require 'active_support/time_with_zone'
+        unpacker = ->(payload) do
+          tv = MessagePack::Timestamp.from_msgpack_ext(payload)
+          ::Time.zone.at(tv.sec, tv.nsec, :nanosecond)
+        end
+        ::ActiveSupport::TimeWithZone.include ::Dynflow::Extensions::MsgPack::Time
+        ::MessagePack::DefaultFactory.register_type(0x01, ActiveSupport::TimeWithZone, packer: MessagePack::Time::Packer, unpacker: unpacker)
+
+        ::DateTime.include ::Dynflow::Extensions::MsgPack::Time
+        ::MessagePack::DefaultFactory.register_type(0x02, DateTime,
+                                                    packer: ->(datetime) { MessagePack::Time::Packer.(datetime.to_time) },
+                                                    unpacker: ->(payload) { unpacker.(payload).to_datetime })
+
+        ::Date.include ::Dynflow::Extensions::MsgPack::Time
+        ::MessagePack::DefaultFactory.register_type(0x03, Date,
+                                                    packer: ->(date) { MessagePack::Time::Packer.(date.to_time) },
+                                                    unpacker: ->(payload) { unpacker.(payload).to_date })
+      rescue LoadError
+        # This is fine
+        nil
+      end
+    end
+  end
+end

data/lib/dynflow/extensions.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Dynflow
+  module Extensions
+    require 'dynflow/extensions/msgpack'
+  end
+end

data/lib/dynflow/persistence.rb

@@ -46,6 +46,16 @@ module Dynflow
       adapter.save_action(execution_plan_id, action.id, action.to_hash)
     end
 
+    def save_output_chunks(execution_plan_id, action_id, chunks)
+      return if chunks.empty?
+
+      adapter.save_output_chunks(execution_plan_id, action_id, chunks)
+    end
+
+    def load_output_chunks(execution_plan_id, action_id)
+      adapter.load_output_chunks(execution_plan_id, action_id)
+    end
+
     def find_execution_plans(options)
       adapter.find_execution_plans(options).map do |execution_plan_hash|
         ExecutionPlan.new_from_hash(execution_plan_hash, @world)

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 require 'sequel'
-require 'multi_json'
+require 'msgpack'
 require 'fileutils'
 require 'csv'
 
+# rubocop:disable Metrics/ClassLength
 module Dynflow
   module PersistenceAdapters
 
@@ -37,12 +38,14 @@ module Dynflow
                                             class action_class execution_plan_uuid queue),
                     envelope:            %w(receiver_id),
                     coordinator_record:  %w(id owner_id class),
-                    delayed:             %w(execution_plan_uuid start_at start_before args_serializer frozen)}
+                    delayed:             %w(execution_plan_uuid start_at start_before args_serializer frozen),
+                    output_chunk:        %w(execution_plan_uuid action_id kind timestamp) }
 
       SERIALIZABLE_COLUMNS = { action:  %w(input output),
                                delayed: %w(serialized_args),
                                execution_plan: %w(run_flow finalize_flow execution_history step_ids),
-                               step:    %w(error children) }
+                               step:    %w(error children),
+                               output_chunk: %w(chunk) }
 
       def initialize(config)
         migrate = true
@@ -83,15 +86,17 @@ module Dynflow
             table(:delayed).where(execution_plan_uuid: uuids).delete
 
             steps = table(:step).where(execution_plan_uuid: uuids)
-            backup_to_csv(steps, backup_dir, 'steps.csv') if backup_dir
+            backup_to_csv(:step, steps, backup_dir, 'steps.csv') if backup_dir
             steps.delete
 
+            output_chunks = table(:output_chunk).where(execution_plan_uuid: uuids).delete
+
             actions = table(:action).where(execution_plan_uuid: uuids)
-            backup_to_csv(actions, backup_dir, 'actions.csv') if backup_dir
+            backup_to_csv(:action, actions, backup_dir, 'actions.csv') if backup_dir
             actions.delete
 
             execution_plans = table(:execution_plan).where(uuid: uuids)
-            backup_to_csv(execution_plans, backup_dir, 'execution_plans.csv') if backup_dir
+            backup_to_csv(:execution_plan, execution_plans, backup_dir, 'execution_plans.csv') if backup_dir
             count += execution_plans.delete
           end
         end
@@ -173,6 +178,18 @@ module Dynflow
         save :action, { execution_plan_uuid: execution_plan_id, id: action_id }, value, with_data: false
       end
 
+      def save_output_chunks(execution_plan_id, action_id, chunks)
+        chunks.each do |chunk|
+          chunk[:execution_plan_uuid] = execution_plan_id
+          chunk[:action_id] = action_id
+          save :output_chunk, {}, chunk, with_data: false
+        end
+      end
+
+      def load_output_chunks(execution_plan_id, action_id)
+        load_records :output_chunk, { execution_plan_uuid: execution_plan_id, action_id: action_id }, [:timestamp, :kind, :chunk]
+      end
+
       def connector_feature!
         unless @additional_responsibilities[:connector]
           raise "The sequel persistence adapter connector feature used but not enabled in additional_features"
@@ -265,14 +282,16 @@ module Dynflow
                  step:                :dynflow_steps,
                  envelope:            :dynflow_envelopes,
                  coordinator_record:  :dynflow_coordinator_records,
-                 delayed:             :dynflow_delayed_plans }
+                 delayed:             :dynflow_delayed_plans,
+                 output_chunk:        :dynflow_output_chunks }
 
       def table(which)
         db[TABLES.fetch(which)]
       end
 
       def initialize_db(db_path)
-        ::Sequel.connect db_path
+        logger = Logger.new($stderr) if ENV['DYNFLOW_SQL_LOG']
+        ::Sequel.connect db_path, logger: logger
       end
 
       def self.migrations_path
@@ -281,10 +300,15 @@ module Dynflow
 
       def prepare_record(table_name, value, base = {}, with_data = true)
         record = base.dup
-        if with_data && table(table_name).columns.include?(:data)
+        has_data_column = table(table_name).columns.include?(:data)
+        if with_data && has_data_column
           record[:data] = dump_data(value)
         else
-          record[:data] = nil
+          if has_data_column
+            record[:data] = nil
+          else
+            record.delete(:data)
+          end
           record.merge! serialize_columns(table_name, value)
         end
 
@@ -339,7 +363,11 @@ module Dynflow
         records = with_retry do
           filtered = table.filter(Utils.symbolize_keys(condition))
           # Filter out requested columns which the table doesn't have, load data just in case
-          filtered = filtered.select(:data, *(table.columns & keys)) unless keys.nil?
+          unless keys.nil?
+            columns = table.columns & keys
+            columns |= [:data] if table.columns.include?(:data)
+            filtered = filtered.select(*columns)
+          end
           filtered.all
         end
         records = records.map { |record| load_data(record, what) }
@@ -355,11 +383,11 @@ module Dynflow
         hash = if record[:data].nil?
                  SERIALIZABLE_COLUMNS.fetch(what, []).each do |key|
                    key = key.to_sym
-                   record[key] = MultiJson.load(record[key]) unless record[key].nil?
+                   record[key] = MessagePack.unpack((record[key])) unless record[key].nil?
                  end
                  record
                else
-                 MultiJson.load(record[:data])
+                 MessagePack.unpack(record[:data])
                end
         Utils.indifferent_hash(hash)
       end
@@ -368,7 +396,7 @@ module Dynflow
         FileUtils.mkdir_p(backup_dir) unless File.directory?(backup_dir)
       end
 
-      def backup_to_csv(dataset, backup_dir, file_name)
+      def backup_to_csv(table_name, dataset, backup_dir, file_name)
         ensure_backup_dir(backup_dir)
         csv_file = File.join(backup_dir, file_name)
         appending = File.exist?(csv_file)
@@ -376,7 +404,12 @@ module Dynflow
         File.open(csv_file, 'a') do |csv|
           csv << columns.to_csv unless appending
           dataset.each do |row|
-            csv << columns.collect { |col| row[col] }.to_csv
+            values = columns.map do |col|
+              value = row[col]
+              value = value.unpack('H*').first if value && SERIALIZABLE_COLUMNS.fetch(table_name, []).include?(col.to_s)
+              value
+            end
+            csv << values.to_csv
           end
         end
         dataset
@@ -394,7 +427,8 @@ module Dynflow
 
       def dump_data(value)
         return if value.nil?
-        MultiJson.dump Type!(value, Hash, Array, Integer)
+        packed = MessagePack.pack(Type!(value, Hash, Array, Integer, String))
+        ::Sequel.blob(packed)
       end
 
       def paginate(data_set, options)
@@ -477,3 +511,4 @@ module Dynflow
     end
   end
 end
+# rubocop:enable Metrics/ClassLength

data/lib/dynflow/persistence_adapters/sequel_migrations/021_create_output_chunks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+Sequel.migration do
+  up do
+    type = database_type
+    create_table(:dynflow_output_chunks) do
+      primary_key :id
+
+      column_properties = if type.to_s.include?('postgres')
+                            {type: :uuid}
+                          else
+                            {type: String, size: 36, fixed: true, null: false}
+                          end
+      foreign_key :execution_plan_uuid, :dynflow_execution_plans, **column_properties
+      index :execution_plan_uuid
+
+      column :action_id, Integer, null: false
+      foreign_key [:execution_plan_uuid, :action_id], :dynflow_actions,
+                  name: :dynflow_output_chunks_execution_plan_uuid_fkey1
+      index [:execution_plan_uuid, :action_id]
+
+      column :chunk, String, text: true
+      column :kind, String
+      column :timestamp, Time, null: false
+    end
+  end
+
+  down do
+    drop_table(:dynflow_output_chunks)
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/022_store_flows_as_msgpack.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'multi_json'
+require 'msgpack'
+
+def table_pkeys(table)
+  case table
+  when :dynflow_execution_plans
+    [:uuid]
+  when :dynflow_actions, :dynflow_steps
+    [:execution_plan_uuid, :id]
+  when :dynflow_coordinator_records
+    [:id, :class]
+  when :dynflow_delayed_plans
+    [:execution_plan_uuid]
+  when :dynflow_envelopes
+    [:id]
+  when :dynflow_output_chunks
+    [:chunk]
+  else
+    raise "Unknown table '#{table}'"
+  end
+end
+
+def conditions_for_row(table, row)
+  row.slice(*table_pkeys(table))
+end
+
+def migrate_table(table, from_names, to_names, new_type)
+  alter_table(table) do
+    to_names.each do |new|
+      add_column new, new_type
+    end
+  end
+
+  relevant_columns = table_pkeys(table) | from_names
+
+  from(table).select(*relevant_columns).each do |row|
+    update = from_names.zip(to_names).reduce({}) do |acc, (from, to)|
+      row[from].nil? ? acc : acc.merge(to => yield(row[from]))
+    end
+    next if update.empty?
+    from(table).where(conditions_for_row(table, row)).update(update)
+  end
+
+  from_names.zip(to_names).each do |old, new|
+    alter_table(table) do
+      drop_column old
+    end
+
+    if database_type == :mysql
+      type = new_type == File ? 'blob' : 'mediumtext'
+      run "ALTER TABLE #{table} CHANGE COLUMN `#{new}` `#{old}` #{type};"
+    else
+      rename_column table, new, old
+    end
+  end
+end
+
+Sequel.migration do
+
+  TABLES = {
+    :dynflow_actions => [:data, :input, :output],
+    :dynflow_coordinator_records => [:data],
+    :dynflow_delayed_plans => [:serialized_args, :data],
+    :dynflow_envelopes => [:data],
+    :dynflow_execution_plans => [:run_flow, :finalize_flow, :execution_history, :step_ids],
+    :dynflow_steps => [:error, :children],
+    :dynflow_output_chunks => [:chunk]
+  }
+
+  up do
+    TABLES.each do |table, columns|
+      new_columns = columns.map { |c| "#{c}_blob" }
+
+      migrate_table table, columns, new_columns, File do |data|
+        ::Sequel.blob(MessagePack.pack(MultiJson.load(data)))
+      end
+    end
+  end
+
+  down do
+    TABLES.each do |table, columns|
+      new_columns = columns.map { |c| c + '_text' }
+      migrate_table table, columns, new_columns, String do |data|
+        MultiJson.dump(MessagePack.unpack(data))
+      end
+    end
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/023_sqlite_workarounds.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+tables = [:dynflow_actions, :dynflow_delayed_plans, :dynflow_steps, :dynflow_output_chunks]
+Sequel.migration do
+  up do
+    if database_type == :sqlite && Gem::Version.new(SQLite3::SQLITE_VERSION) <= Gem::Version.new('3.7.17')
+      tables.each do |table|
+        alter_table(table) { drop_foreign_key [:execution_plan_uuid] }
+      end
+    end
+  end
+
+  down do
+    if database_type == :sqlite && Gem::Version.new(SQLite3::SQLITE_VERSION) <= Gem::Version.new('3.7.17')
+      tables.each do |table|
+        alter_table(table) { add_foreign_key [:execution_plan_uuid], :dynflow_execution_plans }
+      end
+    end
+  end
+end

data/lib/dynflow/serializable.rb

@@ -45,12 +45,12 @@ module Dynflow
       if values.size == 1
         value = values.first
         case value
-        when String, Numeric, Symbol, TrueClass, FalseClass, NilClass, Time
-          value
         when Hash
           value.inject({}) { |h, (k, v)| h.update k => recursive_to_hash(v) }
         when Array
           value.map { |v| recursive_to_hash v }
+        when ->(v) { v.respond_to?(:to_msgpack) }
+          value
         else
           value.to_hash
         end

data/lib/dynflow/testing/dummy_coordinator.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+module Dynflow
+  module Testing
+    class DummyCoordinator
+      def find_records(*args)
+        []
+      end
+    end
+  end
+end

data/lib/dynflow/testing/dummy_planned_action.rb

@@ -15,6 +15,10 @@ module Dynflow
         @plan_input = args
         self
       end
+
+      def run_step_id
+        @run_step_id ||= Testing.get_id
+      end
     end
   end
 end

data/lib/dynflow/testing/dummy_world.rb

@@ -5,7 +5,7 @@ module Dynflow
       extend Mimic
       mimic! World
 
-      attr_reader :clock, :executor, :middleware
+      attr_reader :clock, :executor, :middleware, :coordinator
       attr_accessor :action
 
       def initialize(_config = nil)
@@ -13,6 +13,7 @@ module Dynflow
         @clock          = ManagedClock.new
         @executor       = DummyExecutor.new(self)
         @middleware     = Middleware::World.new
+        @coordinator    = DummyCoordinator.new
       end
 
       def action_logger

data/lib/dynflow/testing.rb

@@ -19,6 +19,7 @@ module Dynflow
 
     require 'dynflow/testing/mimic'
     require 'dynflow/testing/managed_clock'
+    require 'dynflow/testing/dummy_coordinator'
     require 'dynflow/testing/dummy_world'
     require 'dynflow/testing/dummy_executor'
     require 'dynflow/testing/dummy_execution_plan'

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.5.0'
+  VERSION = '1.6.1'
 end

data/lib/dynflow/world.rb

@@ -200,6 +200,14 @@ module Dynflow
       Scheduled[execution_plan.id]
     end
 
+    def plan_elsewhere(action_class, *args)
+      execution_plan = ExecutionPlan.new(self, nil)
+      execution_plan.delay(nil, action_class, {}, *args)
+      plan_request(execution_plan.id)
+
+      Scheduled[execution_plan.id]
+    end
+
     def plan(action_class, *args)
       plan_with_options(action_class: action_class, args: args)
     end
@@ -227,6 +235,10 @@ module Dynflow
       publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time, optional], accepted, false)
     end
 
+    def plan_request(execution_plan_id, done = Concurrent::Promises.resolvable_future)
+      publish_request(Dispatcher::Planning[execution_plan_id], done, false)
+    end
+
     def ping(world_id, timeout, done = Concurrent::Promises.resolvable_future)
       publish_request(Dispatcher::Ping[world_id, true], done, false, timeout)
     end

data/lib/dynflow.rb

@@ -72,6 +72,7 @@ module Dynflow
   require 'dynflow/throttle_limiter'
   require 'dynflow/telemetry'
   require 'dynflow/config'
+  require 'dynflow/extensions'
 
   if defined? ::ActiveJob
     require 'dynflow/active_job/queue_adapter'

data/test/execution_plan_hooks_test.rb

@@ -60,6 +60,18 @@ module Dynflow
         execution_plan_hooks.use :raise_flag_root_only, :on => :stopped
       end
 
+      class PendingAction < ::Dynflow::Action
+        include FlagHook
+
+        execution_plan_hooks.use :raise_flag, :on => :pending
+      end
+
+      class AllTransitionsAction < ::Dynflow::Action
+        include FlagHook
+
+        execution_plan_hooks.use :raise_flag
+      end
+
       class ComposedAction < RootOnlyAction
         def plan
           plan_action(RootOnlyAction)
@@ -161,6 +173,30 @@ module Dynflow
         plan.finished.wait!
         _(Flag.raised_count).must_equal 1
       end
+
+      it 'runs the pending hooks when execution plan is created' do
+        refute Flag.raised?
+        plan = world.trigger(PendingAction)
+        plan.finished.wait!
+        _(Flag.raised_count).must_equal 1
+      end
+
+      it 'runs the pending hooks when execution plan is created' do
+        refute Flag.raised?
+        delay = world.delay(PendingAction, { :start_at => Time.now.utc + 180 })
+        delayed_plan = world.persistence.load_delayed_plan(delay.execution_plan_id)
+        delayed_plan.execution_plan.cancel.each(&:wait)
+        _(Flag.raised_count).must_equal 1
+      end
+
+      it 'runs the hook on every state transition' do
+        refute Flag.raised?
+        plan = world.trigger(AllTransitionsAction)
+        plan.finished.wait!
+        # There should be 5 transitions
+        # nothing -> pending -> planning -> planned -> running -> stopped
+        _(Flag.raised_count).must_equal 5
+      end
     end
   end
 end

data/test/extensions_test.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require_relative 'test_helper'
+require 'active_support/time'
+
+module Dynflow
+  module ExtensionsTest
+    describe 'msgpack extensions' do
+      before do
+        Thread.current[:time_zone] = ActiveSupport::TimeZone['Europe/Prague']
+      end
+      after { Thread.current[:time_zone] = nil }
+
+      it 'allows {de,}serializing Time' do
+        time = Time.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing ActiveSupport::TimeWithZone' do
+        time = Time.zone.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing DateTime' do
+        time = DateTime.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing Date' do
+        date = DateTime.current
+        transformed = MessagePack.unpack(date.to_msgpack)
+        assert_equal transformed, date
+        assert_equal transformed.class, date.class
+      end
+    end
+  end
+end

data/test/future_execution_test.rb

@@ -29,14 +29,17 @@ module Dynflow
         describe 'abstract executor' do
           let(:abstract_delayed_executor) { DelayedExecutors::AbstractCore.new(world) }
 
-          it 'handles wrong plan state' do
+          it 'handles plan in planning state' do
             delayed_plan.execution_plan.state = :planning
             abstract_delayed_executor.send(:process, [delayed_plan], @start_at)
-            _(delayed_plan.execution_plan.state).must_equal :planned
+            _(delayed_plan.execution_plan.state).must_equal :scheduled
+          end
 
+          it 'handles plan in running state' do
             delayed_plan.execution_plan.set_state(:running, true)
             abstract_delayed_executor.send(:process, [delayed_plan], @start_at)
             _(delayed_plan.execution_plan.state).must_equal :running
+            _(world.persistence.load_delayed_plan(delayed_plan.execution_plan_uuid)).must_be :nil?
           end
         end
 
@@ -55,7 +58,7 @@ module Dynflow
 
         it 'delays the action' do
           _(execution_plan.steps.count).must_equal 1
-          _(delayed_plan.start_at).must_be_within_delta(@start_at, 0.5)
+          _(delayed_plan.start_at.to_i).must_equal(@start_at.to_i)
           _(history_names.call(execution_plan)).must_equal ['delay']
         end
 

data/web/views/flow_step.erb

@@ -31,6 +31,7 @@
   <% end %>
   <%= show_action_data("Input:", action.input) %>
   <%= show_action_data("Output:", action.output) %>
+  <%= show_action_data("Chunked output:", action.stored_output_chunks) %>
   <% if step.error %>
     <p>
       <b>Error:</b>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-13 00:00:00.000000000 Z
+date: 2021-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -25,6 +25,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.3
 - !ruby/object:Gem::Dependency
   name: apipie-params
   requirement: !ruby/object:Gem::Requirement
@@ -387,6 +401,7 @@ files:
 - doc/pages/source/projects/index.md
 - docker-compose.yml
 - dynflow.gemspec
+- examples/chunked_output_benchmark.rb
 - examples/clock_benchmark.rb
 - examples/example_helper.rb
 - examples/future_execution.rb
@@ -398,6 +413,7 @@ files:
 - examples/sub_plan_concurrency_control.rb
 - examples/sub_plans.rb
 - examples/termination.rb
+- extras/expand/main.go
 - extras/statsd_mapping.conf
 - lib/dynflow.rb
 - lib/dynflow/action.rb
@@ -470,6 +486,8 @@ files:
 - lib/dynflow/executors/sidekiq/redis_locking.rb
 - lib/dynflow/executors/sidekiq/serialization.rb
 - lib/dynflow/executors/sidekiq/worker_jobs.rb
+- lib/dynflow/extensions.rb
+- lib/dynflow/extensions/msgpack.rb
 - lib/dynflow/flows.rb
 - lib/dynflow/flows/abstract.rb
 - lib/dynflow/flows/abstract_composed.rb
@@ -515,6 +533,9 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/021_create_output_chunks.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/022_store_flows_as_msgpack.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/023_sqlite_workarounds.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb
@@ -536,6 +557,7 @@ files:
 - lib/dynflow/telemetry_adapters/statsd.rb
 - lib/dynflow/testing.rb
 - lib/dynflow/testing/assertions.rb
+- lib/dynflow/testing/dummy_coordinator.rb
 - lib/dynflow/testing/dummy_execution_plan.rb
 - lib/dynflow/testing/dummy_executor.rb
 - lib/dynflow/testing/dummy_planned_action.rb
@@ -578,6 +600,7 @@ files:
 - test/execution_plan_hooks_test.rb
 - test/execution_plan_test.rb
 - test/executor_test.rb
+- test/extensions_test.rb
 - test/flows_test.rb
 - test/future_execution_test.rb
 - test/memory_cosumption_watcher_test.rb
@@ -661,6 +684,7 @@ test_files:
 - test/execution_plan_hooks_test.rb
 - test/execution_plan_test.rb
 - test/executor_test.rb
+- test/extensions_test.rb
 - test/flows_test.rb
 - test/future_execution_test.rb
 - test/memory_cosumption_watcher_test.rb