dynflow 1.4.9 → 1.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f677a2e5d7f119264258b7b36fe187203db60b8b4df4136beb73954857d59bd
-  data.tar.gz: a7cd75185cc99ce4a1e3e18e07565542ebf24bec8109a3291e78c56345bbc0db
+  metadata.gz: 40e7ab36f1ef2943d1cbebccde747a2c89186a620493a74bfefb06a810e8bf13
+  data.tar.gz: c2d4da5a39382f4df50ac0a1664edab9b772f94fc6da4f7f5eec51c24b91e256
 SHA512:
-  metadata.gz: c00baa2c020df6b035aa5a45aaa1b58a08ccc310752f4de0ab54da0207a03439cfde09132a7e8a6f4fe7911058a027e87f5cdb7b2563da035725ba5b482d0ab4
-  data.tar.gz: 4189a3f752642000d4eda05a8d82013d350eb77f7325450dfa01e774672aa33c61f3b32a1c8d5eb75e030d2fae1cb42a1e92da3278278569ba250245b06c3aec
+  metadata.gz: 955818d3401e8641df7ad06ad0307791502437ac3b2549dee9d1161c8dd29d76e83f03b378b6c451547d2e1a00f0b8bbdfa282ea9b6e3fc37005e1fde59ebff6
+  data.tar.gz: 2cb868cab752e95928521d1276bca3091f3ebbd55437e80a82baf980c636880dea3bdb3b345252e2735ffe236960c9d9c6d78a471acc9462c8341581ec91ff98

data/{test/prepare_travis_env.sh → .github/install_dependencies.sh}

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -x
+
 echo "Setting the environment to use ${DB} database"
 
 BUNDLE_CONFIG=.bundle/config
@@ -12,11 +14,9 @@ EOF
 case $DB in
     mysql)
         sed -i 's/:mysql//'g $BUNDLE_CONFIG
-        mysql -e 'create database travis_ci_test;'
     ;;
     postgresql)
         sed -i 's/:postgresql//'g $BUNDLE_CONFIG
-        psql -c 'create database travis_ci_test;' -U postgres
     ;;
     sqlite3)
         # the tests are by default using sqlite3: do nothing

data/.github/workflows/release.yml

@@ -0,0 +1,48 @@
+# workflow name
+name: Generate release-artifacts
+
+# on events
+on:
+  push:
+    tags:
+        - '*'
+
+# workflow tasks
+jobs:
+  generate:
+    name: Generate build artifacts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: olegtarasov/get-tag@v2.1
+        id: tagName
+        with:
+          tagRegex: "v(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
+          tagRegexGroup: 1 # Optional. Default is 1.
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+      - name: Generate build files
+        run: |
+          mkdir -p dist
+          cd extras/expand
+          go build -o ../../dist/dynflow-expand-${VERSION}-x86_64
+        env:
+          VERSION: '${{ steps.tagName.outputs.tag }}'
+      - name: Generate distribution tarball
+        run: |
+          cd extras/expand
+          go mod vendor
+          tar --create \
+            --gzip \
+            --file ../../dist/dynflow-expand-${VERSION}.tar.gz \
+            --transform s/^\./dynflow-expand-${VERSION}/ \
+            .
+        env:
+          VERSION: '${{ steps.tagName.outputs.tag }}'
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: dist/*
+          tag: ${{ github.ref }}
+          overwrite: true
+          file_glob: true

data/.github/workflows/ruby.yml

@@ -0,0 +1,116 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on: [pull_request]
+
+env:
+  TESTOPTS: --verbose
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: Setup
+        run: |
+          gem install bundler
+          bundle install --jobs=3 --retry=3
+      - name: Run rubocop
+        run: bundle exec rubocop
+
+  test:
+    runs-on: ubuntu-latest
+    needs: rubocop
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby_version:
+          - 2.5.0
+          - 2.6.0
+          - 2.7.0
+          - 3.0.0
+        concurrent_ruby_ext:
+          - 'true'
+          - 'false'
+        db:
+          - postgresql
+          - mysql
+          - sqlite3
+        include:
+          - db: postgresql
+            conn_string: postgres://postgres@localhost/travis_ci_test
+          - db: mysql
+            conn_string: mysql2://root@127.0.0.1/travis_ci_test
+          - db: sqlite3
+            conn_string: sqlite:/
+        exclude:
+          - db: mysql
+            ruby_version: 2.5.0
+          - db: mysql
+            ruby_version: 2.6.0
+          - db: mysql
+            ruby_version: 3.0.0
+          - db: mysql
+            concurrent_ruby_ext: 'true'
+          - db: sqlite3
+            ruby_version: 2.5.0
+          - db: sqlite3
+            ruby_version: 2.6.0
+          - db: sqlite3
+            ruby_version: 3.0.0
+          - db: sqlite3
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 2.5.0
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 2.6.0
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 3.0.0
+            concurrent_ruby_ext: 'true'
+
+    services:
+      postgres:
+        image: postgres:12.1
+        ports: ['5432:5432']
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        env:
+          POSTGRES_DB: travis_ci_test
+      mariadb:
+        image: mariadb:10
+        ports: ['3306:3306']
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+          MYSQL_DATABASE: travis_ci_test
+      redis:
+        image: redis:latest
+        ports: ['6379:6379']
+
+    env:
+      DB: ${{ matrix.db }}
+      DB_CONN_STRING: ${{ matrix.conn_string }}
+      CONCURRENT_RUBY_EXT: "${{ matrix.concurrent_ruby_ext }}"
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby_version }}
+    - name: Install dependencies
+      run: .github/install_dependencies.sh
+    - name: Run tests
+      run: bundle exec rake test

data/Gemfile

@@ -35,7 +35,7 @@ end
 
 group :rails do
   gem 'daemons'
-  gem 'rails', '>= 4.2.9'
+  gem 'rails', '>= 4.2.9', '< 7'
   gem 'logging'
 end
 

data/dynflow.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.3.0'
 
   s.add_dependency "multi_json"
+  s.add_dependency "msgpack", '~> 1.3', '>= 1.3.3'
   s.add_dependency "apipie-params"
   s.add_dependency "algebrick", '~> 0.7.0'
   s.add_dependency "concurrent-ruby", '~> 1.1.3'

data/examples/chunked_output_benchmark.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative 'example_helper'
+require 'benchmark'
+
+WORDS = File.readlines('/usr/share/dict/words').map(&:chomp).freeze
+COUNT = WORDS.count
+
+module Common
+  def main_loop
+    if output[:current] < input[:limit]
+      consumed = yield
+      output[:current] += consumed
+      plan_event(nil)
+      suspend
+    end
+  end
+
+  def batch
+    WORDS.drop(output[:current]).take(input[:chunk])
+  end
+end
+
+class Regular < ::Dynflow::Action
+  include Common
+
+  def run(event = nil)
+    output[:current] ||= 0
+    output[:words] ||= []
+
+    main_loop do
+      words = batch
+      output[:words] << words
+      words.count
+    end
+  end
+end
+
+class Chunked < ::Dynflow::Action
+  include Common
+
+  def run(event = nil)
+    output[:current] ||= 0
+
+    main_loop do
+      words = batch
+      output_chunk(words)
+      words.count
+    end
+  end
+end
+
+if $0 == __FILE__
+  ExampleHelper.world.action_logger.level = 4
+  ExampleHelper.world.logger.level = 4
+
+  Benchmark.bm do |bm|
+    bm.report('regular    1000 by    100') { ExampleHelper.world.trigger(Regular, limit: 1000, chunk: 100).finished.wait }
+    bm.report('chunked    1000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 1000, chunk: 100).finished.wait }
+
+    bm.report('regular  10_000 by    100') { ExampleHelper.world.trigger(Regular, limit: 10_000, chunk: 100).finished.wait }
+    bm.report('chunked  10_000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 10_000, chunk: 100).finished.wait }
+
+    bm.report('regular  10_000 by   1000') { ExampleHelper.world.trigger(Regular, limit: 10_000, chunk: 1000).finished.wait }
+    bm.report('chunked  10_000 by   1000') { ExampleHelper.world.trigger(Chunked, limit: 10_000, chunk: 1000).finished.wait }
+
+    bm.report('regular 100_000 by    100') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 100).finished.wait }
+    bm.report('chunked 100_000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 100).finished.wait }
+
+    bm.report('regular 100_000 by   1000') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 1000).finished.wait }
+    bm.report('chunked 100_000 by   1000') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 1000).finished.wait }
+
+    bm.report('regular 100_000 by 10_000') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 10_000).finished.wait }
+    bm.report('chunked 100_000 by 10_000') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 10_000).finished.wait }
+  end
+end

data/extras/expand/Dockerfile

@@ -0,0 +1,9 @@
+FROM alpine:3.15 as builder
+RUN apk add -U go
+ADD ./ work/
+RUN cd /work && \
+    go build
+
+FROM scratch
+COPY --from=builder /work/expand /expand
+CMD ["/expand"]

data/extras/expand/README.md

@@ -0,0 +1,25 @@
+# expand
+
+For a long time, Dynflow's database schema remained stable. To optimize Dynflow
+a bit, we started changing it. One of the changes was changing how we encode
+flows, resulting in flows taking roughly 10x less space.
+
+The other change is not merged yet, but has potentionally bigger impact. We
+store certain columns as JSON objects. The upcoming change uses msgpack instead
+of JSON, resulting in faster encoding and decoding times and smaller storage
+footprint when encoded. The drawback is it is a binary format, so if someone
+dumps the tables from DB as CSV, they won't be human readable.
+
+This tool processes CSV DB dumps and decodes msgpack to json.
+
+## Usage
+
+```shell
+# cat dynflow_execution_plans.csv
+2065cc55-6b03-44b7-947a-e999dcb9057f,,stopped,error,,2021-04-16 09:50:33.826,0,0,,Dynflow::ExecutionPlan,1,\x91a143,\x91a153,\x9283a474696d65ce60795de9a46e616d65a564656c6179a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d34363565316466346561306183a474696d65ce60795de9a46e616d65a774696d656f7574a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d343635653164663465613061,\x9101
+6667374a-beab-4b0b-80c8-3d0392cdde40,,scheduled,pending,,,0,,,Dynflow::ExecutionPlan,1,\x91a143,\x91a153,\x9183a474696d65ce60795de9a46e616d65a564656c6179a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d343635653164663465613061,\x9101
+
+# expand < dynflow_execution_plans.csv
+2065cc55-6b03-44b7-947a-e999dcb9057f,,stopped,error,,2021-04-16 09:50:33.826,0,0,,Dynflow::ExecutionPlan,1,"[""C""]","[""S""]","[{""name"":""delay"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""},{""name"":""timeout"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""}]",[1]
+6667374a-beab-4b0b-80c8-3d0392cdde40,,scheduled,pending,,,0,,,Dynflow::ExecutionPlan,1,"[""C""]","[""S""]","[{""name"":""delay"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""}]",[1]
+```

data/extras/expand/go.mod

@@ -0,0 +1,5 @@
+module github.com/dynflow/dynflow/expand
+
+go 1.15
+
+require github.com/vmihailenco/msgpack/v5 v5.3.5

data/extras/expand/go.sum

@@ -0,0 +1,11 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI=
+github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
+github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

data/extras/expand/main.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/hex"
+	"encoding/json"
+	"github.com/vmihailenco/msgpack/v5"
+	"io"
+	"os"
+)
+
+func main() {
+	reader := csv.NewReader(os.Stdin)
+	defer os.Stdin.Close()
+
+	writer := csv.NewWriter(os.Stdout)
+	defer os.Stdout.Close()
+	defer writer.Flush()
+
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+
+		writer.Write(processRow(record))
+	}
+}
+
+func processRow(record []string) []string {
+	for i, r := range record {
+		if isHexEncoded(r) {
+			record[i] = reencodeField(r)
+		}
+	}
+
+	return record
+}
+
+func isHexEncoded(field string) bool {
+	return len(field) >= 2 && field[0:2] == "\\x"
+}
+
+func reencodeField(field string) string {
+	decoded_bytes, err := hex.DecodeString(field[2:])
+	if err != nil {
+		return field
+	}
+
+	var intermediate interface{}
+	err = msgpack.Unmarshal(decoded_bytes, &intermediate)
+	if err != nil {
+		return field
+	}
+
+	return encode(intermediate)
+}
+
+func encode(data interface{}) string {
+	result, err := json.Marshal(data)
+	if err != nil {
+		panic(err)
+	}
+
+	return string(result)
+}

data/lib/dynflow/action.rb

@@ -105,7 +105,8 @@ module Dynflow
 
     attr_reader :world, :phase, :execution_plan_id, :id, :input,
                 :plan_step_id, :run_step_id, :finalize_step_id,
-                :caller_execution_plan_id, :caller_action_id
+                :caller_execution_plan_id, :caller_action_id,
+                :pending_output_chunks
 
     middleware.use Action::Progress::Calculate
 
@@ -133,6 +134,7 @@ module Dynflow
 
       @input  = OutputReference.deserialize getter.(:input, phase?(Run, Finalize, Present))
       @output = OutputReference.deserialize getter.(:output, false) if phase? Run, Finalize, Present
+      @pending_output_chunks = [] if phase? Run, Finalize
     end
 
     def phase?(*phases)
@@ -169,6 +171,14 @@ module Dynflow
       end
     end
 
+    def output_chunk(chunk, kind: nil, timestamp: Time.now)
+      @pending_output_chunks << { chunk: chunk, kind: kind, timestamp: timestamp }
+    end
+
+    def stored_output_chunks
+      @output_chunks ||= world.persistence.load_output_chunks(@execution_plan_id, @id)
+    end
+
     def caller_action
       phase! Present
       return nil if @caller_action_id

data/lib/dynflow/delayed_executors/abstract_core.rb

@@ -46,24 +46,21 @@ module Dynflow
 
       def process(delayed_plans, check_time)
         processed_plan_uuids = []
+        dispatched_plan_uuids = []
+        planning_locks = world.coordinator.find_records(class: Coordinator::PlanningLock.name)
         delayed_plans.each do |plan|
-          next if plan.frozen
+          next if plan.frozen || locked_for_planning?(planning_locks, plan)
           fix_plan_state(plan)
           with_error_handling do
             if plan.execution_plan.state != :scheduled
               # in case the previous process was terminated after running the plan, but before deleting the delayed plan record.
               @logger.info("Execution plan #{plan.execution_plan_uuid} is expected to be in 'scheduled' state, was '#{plan.execution_plan.state}', skipping")
-            elsif !plan.start_before.nil? && plan.start_before < check_time
-              @logger.debug "Failing plan #{plan.execution_plan_uuid}"
-              plan.timeout
+              processed_plan_uuids << plan.execution_plan_uuid
             else
               @logger.debug "Executing plan #{plan.execution_plan_uuid}"
-              Executors.run_user_code do
-                plan.plan
-                plan.execute
-              end
+              world.plan_request(plan.execution_plan_uuid)
+              dispatched_plan_uuids << plan.execution_plan_uuid
             end
-            processed_plan_uuids << plan.execution_plan_uuid
           end
         end
         world.persistence.delete_delayed_plans(:execution_plan_uuid => processed_plan_uuids) unless processed_plan_uuids.empty?
@@ -72,6 +69,7 @@ module Dynflow
       private
 
       # handle the case, where the process was termintated while planning was in progress before
+      # TODO: Doing execution plan updates in orchestrator is bad
       def fix_plan_state(plan)
         if plan.execution_plan.state == :planning
           @logger.info("Execution plan #{plan.execution_plan_uuid} is expected to be in 'scheduled' state, was '#{plan.execution_plan.state}', auto-fixing")
@@ -79,6 +77,10 @@ module Dynflow
           plan.execution_plan.save
         end
       end
+
+      def locked_for_planning?(planning_locks, plan)
+        planning_locks.any? { |lock| lock.execution_plan_id == plan.execution_plan_uuid }
+      end
     end
   end
 end

data/lib/dynflow/director.rb

@@ -53,7 +53,7 @@ module Dynflow
       end
 
       def self.new_from_hash(hash, *_args)
-        self.new(hash[:execution_plan_id], hash[:queue])
+        self.new(hash[:execution_plan_id], hash[:queue], hash[:sender_orchestrator_id])
       end
     end
 
@@ -108,6 +108,26 @@ module Dynflow
       end
     end
 
+    class PlanningWorkItem < WorkItem
+      def execute
+        plan = world.persistence.load_delayed_plan(execution_plan_id)
+        return if plan.nil? || plan.execution_plan.state != :scheduled
+
+        if !plan.start_before.nil? && plan.start_before < Time.now.utc()
+          plan.timeout
+          return
+        end
+
+        world.coordinator.acquire(Coordinator::PlanningLock.new(world, plan.execution_plan_uuid)) do
+          plan.plan
+        end
+        plan.execute
+      rescue => e
+        world.logger.warn e.message
+        world.logger.debug e.backtrace.join("\n")
+      end
+    end
+
     class FinalizeWorkItem < WorkItem
       attr_reader :finalize_steps_data
 
@@ -147,12 +167,18 @@ module Dynflow
       @logger = world.logger
       @execution_plan_managers = {}
       @rescued_steps = {}
+      @planning_plans = []
     end
 
     def current_execution_plan_ids
       @execution_plan_managers.keys
     end
 
+    def handle_planning(execution_plan_uuid)
+      @planning_plans << execution_plan_uuid
+      [PlanningWorkItem.new(execution_plan_uuid, :default, @world.id)]
+    end
+
     def start_execution(execution_plan_id, finished)
       manager = track_execution_plan(execution_plan_id, finished)
       return [] unless manager
@@ -176,9 +202,16 @@ module Dynflow
     end
 
     def work_finished(work)
-      manager = @execution_plan_managers[work.execution_plan_id]
-      return [] unless manager # skip case when getting event from execution plan that is not running anymore
-      unless_done(manager, manager.what_is_next(work))
+      case work
+      when PlanningWorkItem
+        @planning_plans.delete(work.execution_plan_id)
+        @world.persistence.delete_delayed_plans(:execution_plan_uuid => work.execution_plan_id)
+        []
+      else
+        manager = @execution_plan_managers[work.execution_plan_id]
+        return [] unless manager # skip case when getting event from execution plan that is not running anymore
+        unless_done(manager, manager.what_is_next(work))
+      end
     end
 
     # called when there was an unhandled exception during the execution

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -134,7 +134,7 @@ module Dynflow
       def dispatch_request(request, client_world_id, request_id)
         ignore_unknown = false
         executor_id = match request,
-                            (on ~Execution do |execution|
+                            (on ~Execution | ~Planning do |execution|
                                AnyExecutor
                              end),
                             (on ~Event do |event|

data/lib/dynflow/dispatcher/executor_dispatcher.rb

@@ -9,6 +9,7 @@ module Dynflow
 
       def handle_request(envelope)
         match(envelope.message,
+              on(Planning) { perform_planning(envelope, envelope.message)},
               on(Execution) { perform_execution(envelope, envelope.message) },
               on(Event)     { perform_event(envelope, envelope.message) },
               on(Status)    { get_execution_status(envelope, envelope.message) })
@@ -16,6 +17,13 @@ module Dynflow
 
       protected
 
+      def perform_planning(envelope, planning)
+        @world.executor.plan(planning.execution_plan_id)
+        respond(envelope, Accepted)
+      rescue Dynflow::Error => e
+        respond(envelope, Failed[e.message])
+      end
+
       def perform_execution(envelope, execution)
         allocate_executor(execution.execution_plan_id, envelope.sender_id, envelope.request_id)
         execution_lock = Coordinator::ExecutionLock.new(@world, execution.execution_plan_id, envelope.sender_id, envelope.request_id)

data/lib/dynflow/dispatcher.rb

@@ -14,6 +14,10 @@ module Dynflow
         fields! execution_plan_id: String
       end
 
+      Planning = type do
+        fields! execution_plan_id: String
+      end
+
       Ping = type do
         fields! receiver_id: String,
                 use_cache: type { variants TrueClass, FalseClass }
@@ -24,7 +28,7 @@ module Dynflow
                 execution_plan_id: type { variants String, NilClass }
       end
 
-      variants Event, Execution, Ping, Status
+      variants Event, Execution, Ping, Status, Planning
     end
 
     Response = Algebrick.type do

data/lib/dynflow/execution_history.rb

@@ -12,7 +12,7 @@ module Dynflow
 
     module Event
       def inspect
-        "#{Time.at(time).utc}: #{name}".tap { |s| s << " @ #{world_id}" if world_id }
+        ["#{Time.at(time).utc}: #{name}", world_id].compact.join(' @ ')
       end
     end
 

data/lib/dynflow/execution_plan/hooks.rb

@@ -21,7 +21,7 @@ module Dynflow
         # @param class_name [Class] class of the hook to be run
         # @param on [Symbol, Array<Symbol>] when should the hook be run, one of {HOOK_KINDS}
         # @return [void]
-        def use(class_name, on: HOOK_KINDS)
+        def use(class_name, on: ExecutionPlan.states)
           on = Array[on] unless on.kind_of?(Array)
           validate_kinds!(on)
           if hooks[class_name]

data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb

@@ -31,6 +31,7 @@ module Dynflow
         action = persistence.load_action(self)
         yield action
         persistence.save_action(execution_plan_id, action)
+        persistence.save_output_chunks(execution_plan_id, action.id, action.pending_output_chunks)
         save
 
         return self