dynflow 1.4.7 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffff17987553c70b701f8f7e8b98f5a159c4d0498ccb45b9d41581204edb1b01
-  data.tar.gz: 1399c875ec759cf98a53925caa822e7e03aa8e1e61d9bd2e1957ab69bb07beb8
+  metadata.gz: f660ffddfd3e3c7ea4b8414d2e6f27f48d897d34280ff3bec3c8ab6e360b0bfe
+  data.tar.gz: 46cd643c84f09640a07f00eafc3f4d3feda65746d3c353338d8141277fd77e03
 SHA512:
-  metadata.gz: 51f14bfdcae3d32aaae92716a48668f119327cb8873130dbff7ce9551b40f64d0283037c5a77c5fdced78b612ed8046a449c44ec55e1377d56d247a0cedf3256
-  data.tar.gz: 54787c9fe54caa7227c4122470e398891e161618f5bb02f49332ea2e5a54190dd263e3d46a12691fe47218a81d026d67417b501e07785d6e9eda0aa89f4dd916
+  metadata.gz: 99e59816fbf809403a87bbb1f96ff39958d3d059572de7a66f6205df7c48867baa50d77636d287aebb8c69d73db2cef686860520c6e7db27dbfa8217f5570715
+  data.tar.gz: 315b37f302e1c29eae52d00acabd82b605d88b7982ba3e01c920dd18619967115875ec5a1facffc6299280f01aaa61db8162ce3e02bbbd0ba06a95add1b88d64

data/{test/prepare_travis_env.sh → .github/install_dependencies.sh}

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -x
+
 echo "Setting the environment to use ${DB} database"
 
 BUNDLE_CONFIG=.bundle/config
@@ -12,11 +14,9 @@ EOF
 case $DB in
     mysql)
         sed -i 's/:mysql//'g $BUNDLE_CONFIG
-        mysql -e 'create database travis_ci_test;'
     ;;
     postgresql)
         sed -i 's/:postgresql//'g $BUNDLE_CONFIG
-        psql -c 'create database travis_ci_test;' -U postgres
     ;;
     sqlite3)
         # the tests are by default using sqlite3: do nothing

data/.github/workflows/ruby.yml

@@ -0,0 +1,116 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on: [pull_request]
+
+env:
+  TESTOPTS: --verbose
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: Setup
+        run: |
+          gem install bundler
+          bundle install --jobs=3 --retry=3
+      - name: Run rubocop
+        run: bundle exec rubocop
+
+  test:
+    runs-on: ubuntu-latest
+    needs: rubocop
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby_version:
+          - 2.5.0
+          - 2.6.0
+          - 2.7.0
+          - 3.0.0
+        concurrent_ruby_ext:
+          - 'true'
+          - 'false'
+        db:
+          - postgresql
+          - mysql
+          - sqlite3
+        include:
+          - db: postgresql
+            conn_string: postgres://postgres@localhost/travis_ci_test
+          - db: mysql
+            conn_string: mysql2://root@127.0.0.1/travis_ci_test
+          - db: sqlite3
+            conn_string: sqlite:/
+        exclude:
+          - db: mysql
+            ruby_version: 2.5.0
+          - db: mysql
+            ruby_version: 2.6.0
+          - db: mysql
+            ruby_version: 3.0.0
+          - db: mysql
+            concurrent_ruby_ext: 'true'
+          - db: sqlite3
+            ruby_version: 2.5.0
+          - db: sqlite3
+            ruby_version: 2.6.0
+          - db: sqlite3
+            ruby_version: 3.0.0
+          - db: sqlite3
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 2.5.0
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 2.6.0
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 3.0.0
+            concurrent_ruby_ext: 'true'
+
+    services:
+      postgres:
+        image: postgres:12.1
+        ports: ['5432:5432']
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        env:
+          POSTGRES_DB: travis_ci_test
+      mariadb:
+        image: mariadb:10
+        ports: ['3306:3306']
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+          MYSQL_DATABASE: travis_ci_test
+      redis:
+        image: redis:latest
+        ports: ['6379:6379']
+
+    env:
+      DB: ${{ matrix.db }}
+      DB_CONN_STRING: ${{ matrix.conn_string }}
+      CONCURRENT_RUBY_EXT: "${{ matrix.concurrent_ruby_ext }}"
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby_version }}
+    - name: Install dependencies
+      run: .github/install_dependencies.sh
+    - name: Run tests
+      run: bundle exec rake test

data/dynflow.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.3.0'
 
   s.add_dependency "multi_json"
+  s.add_dependency "msgpack", '~> 1.3.3'
   s.add_dependency "apipie-params"
   s.add_dependency "algebrick", '~> 0.7.0'
   s.add_dependency "concurrent-ruby", '~> 1.1.3'

data/examples/chunked_output_benchmark.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative 'example_helper'
+require 'benchmark'
+
+WORDS = File.readlines('/usr/share/dict/words').map(&:chomp).freeze
+COUNT = WORDS.count
+
+module Common
+  def main_loop
+    if output[:current] < input[:limit]
+      consumed = yield
+      output[:current] += consumed
+      plan_event(nil)
+      suspend
+    end
+  end
+
+  def batch
+    WORDS.drop(output[:current]).take(input[:chunk])
+  end
+end
+
+class Regular < ::Dynflow::Action
+  include Common
+
+  def run(event = nil)
+    output[:current] ||= 0
+    output[:words] ||= []
+
+    main_loop do
+      words = batch
+      output[:words] << words
+      words.count
+    end
+  end
+end
+
+class Chunked < ::Dynflow::Action
+  include Common
+
+  def run(event = nil)
+    output[:current] ||= 0
+
+    main_loop do
+      words = batch
+      output_chunk(words)
+      words.count
+    end
+  end
+end
+
+if $0 == __FILE__
+  ExampleHelper.world.action_logger.level = 4
+  ExampleHelper.world.logger.level = 4
+
+  Benchmark.bm do |bm|
+    bm.report('regular    1000 by    100') { ExampleHelper.world.trigger(Regular, limit: 1000, chunk: 100).finished.wait }
+    bm.report('chunked    1000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 1000, chunk: 100).finished.wait }
+
+    bm.report('regular  10_000 by    100') { ExampleHelper.world.trigger(Regular, limit: 10_000, chunk: 100).finished.wait }
+    bm.report('chunked  10_000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 10_000, chunk: 100).finished.wait }
+
+    bm.report('regular  10_000 by   1000') { ExampleHelper.world.trigger(Regular, limit: 10_000, chunk: 1000).finished.wait }
+    bm.report('chunked  10_000 by   1000') { ExampleHelper.world.trigger(Chunked, limit: 10_000, chunk: 1000).finished.wait }
+
+    bm.report('regular 100_000 by    100') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 100).finished.wait }
+    bm.report('chunked 100_000 by    100') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 100).finished.wait }
+
+    bm.report('regular 100_000 by   1000') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 1000).finished.wait }
+    bm.report('chunked 100_000 by   1000') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 1000).finished.wait }
+
+    bm.report('regular 100_000 by 10_000') { ExampleHelper.world.trigger(Regular, limit: 100_000, chunk: 10_000).finished.wait }
+    bm.report('chunked 100_000 by 10_000') { ExampleHelper.world.trigger(Chunked, limit: 100_000, chunk: 10_000).finished.wait }
+  end
+end

data/extras/expand/main.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/hex"
+	"encoding/json"
+	"github.com/vmihailenco/msgpack"
+	"io"
+	"os"
+)
+
+// dynflow_steps
+// 0                   1  2         3    4     5          6        7         8              9             10              11    12    13           14       15
+// execution_plan_uuid,id,action_id,data,state,started_at,ended_at,real_time,execution_time,progress_done,progress_weight,class,error,action_class,children,queue
+//
+// encoded columns are:
+// 3 - data
+// 12 - error
+// 14 - children
+
+// dynflow_actions
+// 0                   1  2    3                        4                5     6     7      8            9           10
+// execution_plan_uuid,id,data,caller_execution_plan_id,caller_action_id,class,input,output,plan_step_id,run_step_id,finalize_step_id
+//
+// encoded columns are:
+// 2 - data
+// 6 - input
+// 7 - output
+
+// dynflow_execution_plans
+// Without msgpack
+// 0    1    2     3      4          5        6         7              8     9     10       11            12                13                14
+// uuid,data,state,result,started_at,ended_at,real_time,execution_time,label,class,run_flow,finalize_flow,execution_history,root_plan_step_id,step_ids
+
+// With msgpack
+// 0    1    2     3      4          5        6         7              8     9     10                11       12            13                14
+// uuid,data,state,result,started_at,ended_at,real_time,execution_time,label,class,root_plan_step_id,run_flow,finalize_flow,execution_history,step_ids
+//
+// 1 - data
+// 11 - run_flow
+// 12 - finalize_flow
+// 13 - execution_history
+// 14 - step_ids
+
+func main() {
+	reader := csv.NewReader(os.Stdin)
+	writer := csv.NewWriter(os.Stdout)
+	defer writer.Flush()
+
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+
+		writer.Write(processRow(record))
+	}
+}
+
+func processRow(record []string) []string {
+	// Execution plan exports have 15 fields, other exports have different counts
+	if len(record) == 15 {
+		record = expandExecutionPlan(record)
+	}
+
+	for i, r := range record {
+		record[i] = reencodeField(r)
+	}
+
+	return record
+}
+
+func expandExecutionPlan(record []string) []string {
+	var flow_columns [2]int
+
+	// The step_ids field should be a safe indicator
+	if isHexEncoded(record[14]) {
+		flow_columns = [...]int{11, 12}
+	} else {
+		flow_columns = [...]int{10, 11}
+	}
+
+	for _, i := range flow_columns {
+		record[i] = expandFlow(record[i])
+	}
+	return record
+}
+
+func isHexEncoded(field string) bool {
+	return len(field) >= 2 && field[0:2] == "\\x"
+}
+
+func reencodeField(field string) string {
+	decoded, err := decode(field)
+	if err != nil {
+		return field
+	}
+
+	return encode(decoded)
+}
+
+func decode(field string) (interface{}, error) {
+	var intermediate interface{}
+	bytes := []byte(field)
+
+	if isHexEncoded(field) {
+		decoded_bytes, err := hex.DecodeString(field[2:])
+		if err != nil {
+			return "", err
+		}
+
+		err = msgpack.Unmarshal(decoded_bytes, &intermediate)
+		if err != nil {
+			return "", err
+		}
+
+		return intermediate, nil
+	}
+
+	err := json.Unmarshal(bytes, &intermediate)
+	if err != nil {
+		return "", err
+	}
+
+	return intermediate, nil
+}
+
+func encode(data interface{}) string {
+	result, err := json.Marshal(data)
+	if err != nil {
+		panic(err)
+	}
+
+	return string(result)
+}
+
+func expandFlow(field string) string {
+	intermediate, err := decode(field)
+	if err != nil {
+		return field
+	}
+
+	var result map[string]interface{}
+	switch intermediate.(type) {
+	// old style hash
+	case map[string]interface{}:
+		result = intermediate.(map[string]interface{})
+	// newer compact S-expression like representation
+	case []interface{}, float64:
+		result = expandCompactFlow(intermediate)
+	}
+
+	return encode(result)
+}
+
+func expandCompactFlow(flow interface{}) map[string]interface{} {
+	result := make(map[string]interface{})
+	switch flow.(type) {
+	case []interface{}:
+		switch flow.([]interface{})[0] {
+		case "S":
+			result["class"] = "Dynflow::Flows::Sequence"
+		case "C":
+			result["class"] = "Dynflow::Flows::Concurrence"
+		default:
+			panic("Unknown flow type")
+		}
+		var subflows []interface{}
+		for subflow := range flow.([]interface{})[1:] {
+			subflows = append(subflows, expandCompactFlow(subflow))
+		}
+		result["flows"] = subflows
+	case float64, int:
+		result["class"] = "Dynflow::Flows::Atom"
+		result["step_id"] = flow
+	default:
+		panic("Unknown flow type")
+	}
+	return result
+}

data/lib/dynflow/action/suspended.rb

@@ -9,14 +9,14 @@ module Dynflow
       @step_id           = action.run_step_id
     end
 
-    def plan_event(event, time, sent = Concurrent::Promises.resolvable_future)
-      @world.plan_event(execution_plan_id, step_id, event, time, sent)
+    def plan_event(event, time, sent = Concurrent::Promises.resolvable_future, optional: false)
+      @world.plan_event(execution_plan_id, step_id, event, time, sent, optional: optional)
     end
 
-    def event(event, sent = Concurrent::Promises.resolvable_future)
+    def event(event, sent = Concurrent::Promises.resolvable_future, optional: false)
       # TODO: deprecate 2 levels backtrace (to know it's called from clock or internaly)
       # remove lib/dynflow/clock.rb ClockReference#ping branch condition on removal.
-      plan_event(event, nil, sent)
+      plan_event(event, nil, sent, optional: optional)
     end
 
     def <<(event = nil)

data/lib/dynflow/action/timeouts.rb

@@ -7,8 +7,8 @@ module Dynflow
       fail("Timeout exceeded.")
     end
 
-    def schedule_timeout(seconds)
-      plan_event(Timeout, seconds)
+    def schedule_timeout(seconds, optional: false)
+      plan_event(Timeout, seconds, optional: optional)
     end
  end
 end

data/lib/dynflow/action.rb

@@ -93,7 +93,8 @@ module Dynflow
       fields! execution_plan_id: String,
               step_id:           Integer,
               event:             Object,
-              time:              type { variants Time, NilClass }
+              time:              type { variants Time, NilClass },
+              optional:          Algebrick::Types::Boolean
     end
 
     def self.constantize(action_name)
@@ -104,7 +105,8 @@ module Dynflow
 
     attr_reader :world, :phase, :execution_plan_id, :id, :input,
                 :plan_step_id, :run_step_id, :finalize_step_id,
-                :caller_execution_plan_id, :caller_action_id
+                :caller_execution_plan_id, :caller_action_id,
+                :pending_output_chunks
 
     middleware.use Action::Progress::Calculate
 
@@ -132,6 +134,7 @@ module Dynflow
 
       @input  = OutputReference.deserialize getter.(:input, phase?(Run, Finalize, Present))
       @output = OutputReference.deserialize getter.(:output, false) if phase? Run, Finalize, Present
+      @pending_output_chunks = [] if phase? Run, Finalize
     end
 
     def phase?(*phases)
@@ -168,6 +171,14 @@ module Dynflow
       end
     end
 
+    def output_chunk(chunk, kind: nil, timestamp: Time.now)
+      @pending_output_chunks << { chunk: chunk, kind: kind, timestamp: timestamp }
+    end
+
+    def stored_output_chunks
+      @output_chunks ||= world.persistence.load_output_chunks(@execution_plan_id, @id)
+    end
+
     def caller_action
       phase! Present
       return nil if @caller_action_id
@@ -332,9 +343,9 @@ module Dynflow
 
     # Plan an +event+ to be send to the action defined by +action+, what defaults to be self.
     # if +time+ is not passed, event is sent as soon as possible.
-    def plan_event(event, time = nil, execution_plan_id: self.execution_plan_id, step_id: self.run_step_id)
+    def plan_event(event, time = nil, execution_plan_id: self.execution_plan_id, step_id: self.run_step_id, optional: false)
       time = @world.clock.current_time + time if time.is_a?(Numeric)
-      delayed_events << DelayedEvent[execution_plan_id, step_id, event, time]
+      delayed_events << DelayedEvent[execution_plan_id, step_id, event, time, optional]
     end
 
     def delayed_events

data/lib/dynflow/actor.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 module Dynflow
 
+  FULL_BACKTRACE = %w[1 y yes].include?((ENV['DYNFLOW_FULL_BACKTRACE'] || '').downcase)
+  BACKTRACE_LIMIT = begin
+                      limit = ENV['DYNFLOW_BACKTRACE_LIMIT'].to_i
+                      limit.zero? ? nil : limit
+                    end
+
   module MethodicActor
     def on_message(message)
       method, *args = message
@@ -44,7 +50,11 @@ module Dynflow
       include LogWithFullBacktrace
 
       def on_envelope(envelope)
-        Actor::BacktraceCollector.with_backtrace(envelope.origin_backtrace) { super }
+        if FULL_BACKTRACE
+          Actor::BacktraceCollector.with_backtrace(envelope.origin_backtrace) { super }
+        else
+          super
+        end
       end
     end
 
@@ -83,9 +93,15 @@ module Dynflow
 
         # takes an array of backtrace lines and replaces each chunk
         def filter_backtrace(backtrace)
-          backtrace.map { |line| filter_line(line) }
-            .chunk_while { |l1, l2| l1 == l2}
-            .map(&:first)
+          trace = backtrace.map { |line| filter_line(line) }
+                           .chunk_while { |l1, l2| l1 == l2}
+                           .map(&:first)
+          if BACKTRACE_LIMIT
+            count = trace.count
+            trace = trace.take(BACKTRACE_LIMIT)
+            trace << "[ backtrace omitted #{count - BACKTRACE_LIMIT} lines ]" if trace.count < count
+          end
+          trace
         end
       end
     end

data/lib/dynflow/clock.rb

@@ -114,11 +114,11 @@ module Dynflow
       Time.now
     end
 
-    def ping(who, time, with_what = nil, where = :<<)
+    def ping(who, time, with_what = nil, where = :<<, optional: false)
       Type! time, Time, Numeric
       time  = current_time + time if time.is_a? Numeric
       if who.is_a?(Action::Suspended)
-        who.plan_event(with_what, time)
+        who.plan_event(with_what, time, optional: optional)
       else
         timer = Clock::Timer[who, time, with_what.nil? ? Algebrick::Types::None : Some[Object][with_what], where]
         self.tell([:add_timer, timer])

data/lib/dynflow/delayed_executors/abstract_core.rb

@@ -46,24 +46,21 @@ module Dynflow
 
       def process(delayed_plans, check_time)
         processed_plan_uuids = []
+        dispatched_plan_uuids = []
+        planning_locks = world.coordinator.find_records(class: Coordinator::PlanningLock.name)
         delayed_plans.each do |plan|
-          next if plan.frozen
+          next if plan.frozen || locked_for_planning?(planning_locks, plan)
           fix_plan_state(plan)
           with_error_handling do
             if plan.execution_plan.state != :scheduled
               # in case the previous process was terminated after running the plan, but before deleting the delayed plan record.
               @logger.info("Execution plan #{plan.execution_plan_uuid} is expected to be in 'scheduled' state, was '#{plan.execution_plan.state}', skipping")
-            elsif !plan.start_before.nil? && plan.start_before < check_time
-              @logger.debug "Failing plan #{plan.execution_plan_uuid}"
-              plan.timeout
+              processed_plan_uuids << plan.execution_plan_uuid
             else
               @logger.debug "Executing plan #{plan.execution_plan_uuid}"
-              Executors.run_user_code do
-                plan.plan
-                plan.execute
-              end
+              world.plan_request(plan.execution_plan_uuid)
+              dispatched_plan_uuids << plan.execution_plan_uuid
             end
-            processed_plan_uuids << plan.execution_plan_uuid
           end
         end
         world.persistence.delete_delayed_plans(:execution_plan_uuid => processed_plan_uuids) unless processed_plan_uuids.empty?
@@ -72,6 +69,7 @@ module Dynflow
       private
 
       # handle the case, where the process was termintated while planning was in progress before
+      # TODO: Doing execution plan updates in orchestrator is bad
       def fix_plan_state(plan)
         if plan.execution_plan.state == :planning
           @logger.info("Execution plan #{plan.execution_plan_uuid} is expected to be in 'scheduled' state, was '#{plan.execution_plan.state}', auto-fixing")
@@ -79,6 +77,10 @@ module Dynflow
           plan.execution_plan.save
         end
       end
+
+      def locked_for_planning?(planning_locks, plan)
+        planning_locks.any? { |lock| lock.execution_plan_id == plan.execution_plan_uuid }
+      end
     end
   end
 end

data/lib/dynflow/director/running_steps_manager.rb

@@ -20,8 +20,8 @@ module Dynflow
       def terminate
         pending_work = @work_items.clear.values.flatten(1)
         pending_work.each do |w|
-          if EventWorkItem === w && w.event.result
-            w.event.result.reject UnprocessableEvent.new("dropping due to termination")
+          finish_event_result(w) do |result|
+            result.reject UnprocessableEvent.new("dropping due to termination")
           end
         end
       end