dynflow 1.9.3 → 2.0.0

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -39,7 +39,8 @@ module Dynflow
                     envelope:            %w(receiver_id),
                     coordinator_record:  %w(id owner_id class),
                     delayed:             %w(execution_plan_uuid start_at start_before args_serializer frozen),
-                    output_chunk:        %w(execution_plan_uuid action_id kind timestamp) }
+                    output_chunk:        %w(execution_plan_uuid action_id kind timestamp),
+                    execution_plan_dependency: %w(execution_plan_uuid blocked_by_uuid) }
 
       SERIALIZABLE_COLUMNS = { action:  %w(input output),
                                delayed: %w(serialized_args),
@@ -71,20 +72,23 @@ module Dynflow
             paginate(table(table_name), options),
             options),
           options[:filters])
-        data_set.all.map { |record| execution_plan_column_map(load_data(record, table_name)) }
+        records = with_retry { data_set.all }
+        records.map { |record| execution_plan_column_map(load_data(record, table_name)) }
       end
 
       def find_execution_plan_counts(options = {})
-        filter(:execution_plan, table(:execution_plan), options[:filters]).count
+        with_retry { filter(:execution_plan, table(:execution_plan), options[:filters]).count }
       end
 
       def find_execution_plan_counts_after(timestamp, options = {})
-        filter(:execution_plan, table(:execution_plan), options[:filters]).filter(::Sequel.lit('ended_at >= ?', timestamp)).count
+        with_retry { filter(:execution_plan, table(:execution_plan), options[:filters]).filter(::Sequel.lit('ended_at >= ?', timestamp)).count }
       end
 
       def find_execution_plan_statuses(options)
-        plans = filter(:execution_plan, table(:execution_plan), options[:filters])
-                .select(:uuid, :state, :result)
+        plans = with_retry do
+          filter(:execution_plan, table(:execution_plan), options[:filters])
+            .select(:uuid, :state, :result)
+        end
 
         plans.each_with_object({}) do |current, acc|
           uuid = current.delete(:uuid)
@@ -94,27 +98,29 @@ module Dynflow
 
       def delete_execution_plans(filters, batch_size = 1000, backup_dir = nil)
         count = 0
-        filter(:execution_plan, table(:execution_plan), filters).each_slice(batch_size) do |plans|
-          uuids = plans.map { |p| p.fetch(:uuid) }
-          @db.transaction do
-            table(:delayed).where(execution_plan_uuid: uuids).delete
+        with_retry do
+          filter(:execution_plan, table(:execution_plan), filters).each_slice(batch_size) do |plans|
+            uuids = plans.map { |p| p.fetch(:uuid) }
+            @db.transaction do
+              table(:delayed).where(execution_plan_uuid: uuids).delete
 
-            steps = table(:step).where(execution_plan_uuid: uuids)
-            backup_to_csv(:step, steps, backup_dir, 'steps.csv') if backup_dir
-            steps.delete
+              steps = table(:step).where(execution_plan_uuid: uuids)
+              backup_to_csv(:step, steps, backup_dir, 'steps.csv') if backup_dir
+              steps.delete
 
-            table(:output_chunk).where(execution_plan_uuid: uuids).delete
+              output_chunks = table(:output_chunk).where(execution_plan_uuid: uuids).delete
 
-            actions = table(:action).where(execution_plan_uuid: uuids)
-            backup_to_csv(:action, actions, backup_dir, 'actions.csv') if backup_dir
-            actions.delete
+              actions = table(:action).where(execution_plan_uuid: uuids)
+              backup_to_csv(:action, actions, backup_dir, 'actions.csv') if backup_dir
+              actions.delete
 
-            execution_plans = table(:execution_plan).where(uuid: uuids)
-            backup_to_csv(:execution_plan, execution_plans, backup_dir, 'execution_plans.csv') if backup_dir
-            count += execution_plans.delete
+              execution_plans = table(:execution_plan).where(uuid: uuids)
+              backup_to_csv(:execution_plan, execution_plans, backup_dir, 'execution_plans.csv') if backup_dir
+              count += execution_plans.delete
+            end
           end
+          return count
         end
-        return count
       end
 
       def load_execution_plan(execution_plan_id)
@@ -127,10 +133,12 @@ module Dynflow
 
       def delete_delayed_plans(filters, batch_size = 1000)
         count = 0
-        filter(:delayed, table(:delayed), filters).each_slice(batch_size) do |plans|
-          uuids = plans.map { |p| p.fetch(:execution_plan_uuid) }
-          @db.transaction do
-            count += table(:delayed).where(execution_plan_uuid: uuids).delete
+        with_retry do
+          filter(:delayed, table(:delayed), filters).each_slice(batch_size) do |plans|
+            uuids = plans.map { |p| p.fetch(:execution_plan_uuid) }
+            @db.transaction do
+              count += table(:delayed).where(execution_plan_uuid: uuids).delete
+            end
           end
         end
         count
@@ -138,19 +146,43 @@ module Dynflow
 
       def find_old_execution_plans(age)
         table_name = :execution_plan
-        table(table_name)
-          .where(::Sequel.lit('ended_at <= ? AND state = ?', age, 'stopped'))
-          .all.map { |plan| execution_plan_column_map(load_data plan, table_name) }
+        records = with_retry do
+          table(table_name)
+            .where(::Sequel.lit('ended_at <= ? AND state = ?', age, 'stopped'))
+            .all
+        end
+        records.map { |plan| execution_plan_column_map(load_data plan, table_name) }
+      end
+
+      def find_execution_plan_dependencies(execution_plan_id)
+        table(:execution_plan_dependency)
+          .where(execution_plan_uuid: execution_plan_id)
+          .select_map(:blocked_by_uuid)
+      end
+
+      def find_blocked_execution_plans(execution_plan_id)
+        table(:execution_plan_dependency)
+          .where(blocked_by_uuid: execution_plan_id)
+          .select_map(:execution_plan_uuid)
       end
 
-      def find_past_delayed_plans(time)
+      def find_ready_delayed_plans(time)
         table_name = :delayed
-        table(table_name)
-          .where(::Sequel.lit('start_at <= ? OR (start_before IS NOT NULL AND start_before <= ?)', time, time))
-          .where(:frozen => false)
-          .order_by(:start_at)
-          .all
-          .map { |plan| load_data(plan, table_name) }
+        # Subquery to find delayed plans that have at least one non-stopped dependency
+        plans_with_unfinished_deps = table(:execution_plan_dependency)
+                                     .join(TABLES[:execution_plan], uuid: :blocked_by_uuid)
+                                     .where(::Sequel.~(state: 'stopped'))
+                                     .select(:execution_plan_uuid)
+
+        records = with_retry do
+          table(table_name)
+            .where(::Sequel.lit('start_at IS NULL OR (start_at <= ? OR (start_before IS NOT NULL AND start_before <= ?))', time, time))
+            .where(:frozen => false)
+            .exclude(execution_plan_uuid: plans_with_unfinished_deps)
+            .order_by(:start_at)
+            .all
+        end
+        records.map { |plan| load_data(plan, table_name) }
       end
 
       def load_delayed_plan(execution_plan_id)
@@ -163,6 +195,10 @@ module Dynflow
         save :delayed, { execution_plan_uuid: execution_plan_id }, value, with_data: false
       end
 
+      def chain_execution_plan(first, second)
+        save :execution_plan_dependency, {}, { execution_plan_uuid: second, blocked_by_uuid: first }, with_data: false
+      end
+
       def load_step(execution_plan_id, step_id)
         load :step, execution_plan_uuid: execution_plan_id, id: step_id
       end
@@ -205,7 +241,9 @@ module Dynflow
       end
 
       def delete_output_chunks(execution_plan_id, action_id)
-        filter(:output_chunk, table(:output_chunk), { execution_plan_uuid: execution_plan_id, action_id: action_id }).delete
+        with_retry do
+          filter(:output_chunk, table(:output_chunk), { execution_plan_uuid: execution_plan_id, action_id: action_id }).delete
+        end
       end
 
       def connector_feature!
@@ -221,28 +259,30 @@ module Dynflow
 
       def pull_envelopes(receiver_id)
         connector_feature!
-        db.transaction do
-          data_set = table(:envelope).where(receiver_id: receiver_id).all
-          envelopes = data_set.map { |record| load_data(record) }
+        with_retry do
+          db.transaction do
+            data_set = table(:envelope).where(receiver_id: receiver_id).all
+            envelopes = data_set.map { |record| load_data(record) }
 
-          table(:envelope).where(id: data_set.map { |d| d[:id] }).delete
-          return envelopes
+            table(:envelope).where(id: data_set.map { |d| d[:id] }).delete
+            return envelopes
+          end
         end
       end
 
       def push_envelope(envelope)
         connector_feature!
-        table(:envelope).insert(prepare_record(:envelope, envelope))
+        with_retry { table(:envelope).insert(prepare_record(:envelope, envelope)) }
       end
 
       def prune_envelopes(receiver_ids)
         connector_feature!
-        table(:envelope).where(receiver_id: receiver_ids).delete
+        with_retry { table(:envelope).where(receiver_id: receiver_ids).delete }
       end
 
       def prune_undeliverable_envelopes
         connector_feature!
-        table(:envelope).where(receiver_id: table(:coordinator_record).select(:id)).invert.delete
+        with_retry { table(:envelope).where(receiver_id: table(:coordinator_record).select(:id)).invert.delete }
       end
 
       def coordinator_feature!
@@ -263,7 +303,7 @@ module Dynflow
 
       def delete_coordinator_record(class_name, record_id)
         coordinator_feature!
-        table(:coordinator_record).where(class: class_name, id: record_id).delete
+        with_retry { table(:coordinator_record).where(class: class_name, id: record_id).delete }
       end
 
       def find_coordinator_records(options)
@@ -275,7 +315,9 @@ module Dynflow
         if exclude_owner_id
           data_set = data_set.exclude(:owner_id => exclude_owner_id)
         end
-        data_set.all.map { |record| load_data(record) }
+        with_retry do
+          data_set.all.map { |record| load_data(record) }
+        end
       end
 
       def to_hash
@@ -301,7 +343,8 @@ module Dynflow
                  envelope:            :dynflow_envelopes,
                  coordinator_record:  :dynflow_coordinator_records,
                  delayed:             :dynflow_delayed_plans,
-                 output_chunk:        :dynflow_output_chunks }
+                 output_chunk:        :dynflow_output_chunks,
+                 execution_plan_dependency: :dynflow_execution_plan_dependencies }
 
       def table(which)
         db[TABLES.fetch(which)]
@@ -444,7 +487,7 @@ module Dynflow
       end
 
       def delete(what, condition)
-        table(what).where(Utils.symbolize_keys(condition)).delete
+        with_retry { table(what).where(Utils.symbolize_keys(condition)).delete }
       end
 
       def extract_metadata(what, value)

data/lib/dynflow/persistence_adapters/sequel_migrations/025_create_execution_plan_dependencies.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    type = database_type
+    create_table(:dynflow_execution_plan_dependencies) do
+      column_properties = if type.to_s.include?('postgres')
+                            { type: :uuid }
+                          else
+                            { type: String, size: 36, fixed: true, null: false }
+                          end
+      foreign_key :execution_plan_uuid, :dynflow_execution_plans, on_delete: :cascade, **column_properties
+      foreign_key :blocked_by_uuid, :dynflow_execution_plans, on_delete: :cascade, **column_properties
+      index :blocked_by_uuid
+      index :execution_plan_uuid
+    end
+  end
+
+  down do
+    drop_table(:dynflow_execution_plan_dependencies)
+  end
+end

data/lib/dynflow/rails/daemon.rb

@@ -23,23 +23,32 @@ module Dynflow
         @daemons_class || ::Daemons
       end
 
+      def stdout
+        STDOUT
+      end
+
+      def stderr
+        STDERR
+      end
+
       # Load the Rails environment and initialize the executor in this thread.
       def run(rails_root = Dir.pwd, options = {})
-        STDOUT.puts('Starting Rails environment')
+        stdout.puts('Starting Rails environment')
         rails_env_file = File.expand_path('./config/environment.rb', rails_root)
         unless File.exist?(rails_env_file)
           raise "#{rails_root} doesn't seem to be a Rails root directory"
         end
 
-        STDERR.puts("Starting dynflow with the following options: #{options}")
+        stderr.puts("Starting dynflow with the following options: #{options}")
 
         ::Rails.application.dynflow.executor!
 
         if options[:memory_limit] && options[:memory_limit].to_i > 0
           ::Rails.application.dynflow.config.on_init do |world|
+            stdout_cap = stdout
             memory_watcher = initialize_memory_watcher(world, options[:memory_limit], options)
             world.terminated.on_resolution do
-              STDOUT.puts("World has been terminated")
+              stdout_cap.puts("World has been terminated")
               memory_watcher = nil # the object can be disposed
             end
           end
@@ -48,10 +57,10 @@ module Dynflow
         require rails_env_file
         ::Rails.application.dynflow.initialize!
         world_id = ::Rails.application.dynflow.world.id
-        STDOUT.puts("Everything ready for world: #{world_id}")
+        stdout.puts("Everything ready for world: #{world_id}")
         sleep
       ensure
-        STDOUT.puts('Exiting')
+        stdout.puts('Exiting')
       end
 
       # run the executor as a daemon
@@ -68,7 +77,7 @@ module Dynflow
           raise "Command exptected to be 'start', 'stop', 'restart', 'run', was #{command.inspect}"
         end
 
-        STDOUT.puts("Dynflow Executor: #{command} in progress")
+        stdout.puts("Dynflow Executor: #{command} in progress")
 
         options[:executors_count].times do
           daemons_class.run_proc(
@@ -79,7 +88,7 @@ module Dynflow
               ::Logging.reopen
               run(options[:rails_root], options)
             rescue => e
-              STDERR.puts e.message
+              stderr.puts e.message
               ::Rails.logger.fatal('Failed running Dynflow daemon')
               ::Rails.logger.fatal(e)
               exit 1

data/lib/dynflow/testing.rb

@@ -5,7 +5,7 @@ module Dynflow
     extend Algebrick::TypeCheck
 
     def self.logger_adapter
-      @logger_adapter || LoggerAdapters::Simple.new($stdout, 1)
+      @logger_adapter || LoggerAdapters::Simple.new('test.log', 1)
     end
 
     def self.logger_adapter=(adapter)

data/lib/dynflow/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dynflow
-  VERSION = '1.9.3'
+  VERSION = '2.0.0'
 end

data/lib/dynflow/world.rb

@@ -202,6 +202,16 @@ module Dynflow
       Scheduled[execution_plan.id]
     end
 
+    def chain(plan_uuids, action_class, *args)
+      plan_uuids = [plan_uuids] unless plan_uuids.is_a? Array
+      result = delay_with_options(action_class: action_class, args: args, delay_options: { frozen: true })
+      plan_uuids.each do |plan_uuid|
+        persistence.chain_execution_plan(plan_uuid, result.execution_plan_id)
+      end
+      persistence.set_delayed_plan_frozen(result.execution_plan_id, false)
+      result
+    end
+
     def plan_elsewhere(action_class, *args)
       execution_plan = ExecutionPlan.new(self, nil)
       execution_plan.delay(nil, action_class, {}, *args)
@@ -325,17 +335,7 @@ module Dynflow
             logger.info "start terminating throttle_limiter..."
             throttle_limiter.terminate.wait(termination_timeout)
 
-            if executor
-              connector.stop_receiving_new_work(self, termination_timeout)
-
-              logger.info "start terminating executor..."
-              executor.terminate.wait(termination_timeout)
-
-              logger.info "start terminating executor dispatcher..."
-              executor_dispatcher_terminated = Concurrent::Promises.resolvable_future
-              executor_dispatcher.ask([:start_termination, executor_dispatcher_terminated])
-              executor_dispatcher_terminated.wait(termination_timeout)
-            end
+            terminate_executor
 
             logger.info "start terminating client dispatcher..."
             client_dispatcher_terminated = Concurrent::Promises.resolvable_future
@@ -350,7 +350,11 @@ module Dynflow
               clock.ask(:terminate!).wait(termination_timeout)
             end
 
-            coordinator.delete_world(registered_world, true)
+            begin
+              coordinator.delete_world(registered_world, true)
+            rescue Dynflow::Errors::FatalPersistenceError => e
+              nil
+            end
             @terminated.resolve
             true
           rescue => e
@@ -361,7 +365,10 @@ module Dynflow
           termination_future.wait(termination_timeout)
         end.on_resolution do
           @terminated.resolve
-          Thread.new { Kernel.exit } if @exit_on_terminate.true?
+          Thread.new do
+            logger.info 'World terminated, exiting.'
+            Kernel.exit if @exit_on_terminate.true?
+          end
         end
       end
     end
@@ -395,6 +402,20 @@ module Dynflow
       initialized.wait
       return actor
     end
+
+    def terminate_executor
+      return unless executor
+
+      connector.stop_receiving_new_work(self, termination_timeout)
+
+      logger.info "start terminating executor..."
+      executor.terminate.wait(termination_timeout)
+
+      logger.info "start terminating executor dispatcher..."
+      executor_dispatcher_terminated = Concurrent::Promises.resolvable_future
+      executor_dispatcher.ask([:start_termination, executor_dispatcher_terminated])
+      executor_dispatcher_terminated.wait(termination_timeout)
+    end
   end
   # rubocop:enable Metrics/ClassLength
 end

data/lib/dynflow.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'apipie-params'
 require 'algebrick'
 require 'set'
 require 'base64'

data/test/action_test.rb

@@ -135,7 +135,7 @@ module Dynflow
         plan = create_and_plan_action(PlanEventedAction, { time: 0.5 })
         action = run_action plan
 
-        _(action.output[:status]).must_equal nil
+        assert_nil action.output[:status]
         _(action.world.clock.pending_pings.first).wont_be_nil
         _(action.state).must_equal :suspended
 
@@ -150,7 +150,7 @@ module Dynflow
         plan = create_and_plan_action(PlanEventedAction, { time: nil })
         action = run_action plan
 
-        _(action.output[:status]).must_equal nil
+        assert_nil action.output[:status]
         _(action.world.clock.pending_pings.first).must_be_nil
         _(action.world.executor.events_to_process.first).wont_be_nil
         _(action.state).must_equal :suspended
@@ -905,7 +905,7 @@ module Dynflow
 
       it 'collects and drops output chunks' do
         action = create_and_plan_action(OutputChunkAction)
-        _(action.pending_output_chunks).must_equal nil
+        assert_nil action.pending_output_chunks
 
         action = run_action(action)
         _(action.pending_output_chunks.count).must_equal 1

data/test/bats/helpers/common.bash

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Common helper functions for bats tests
+
+# Get the project root directory
+get_project_root() {
+  local dir="${BATS_TEST_DIRNAME}"
+  while [ "${dir}" != "/" ]; do
+    if [ -f "${dir}/dynflow.gemspec" ]; then
+      echo "${dir}"
+      return 0
+    fi
+    dir="$(dirname "${dir}")"
+  done
+  echo "ERROR: Could not find project root" >&2
+  return 1
+}
+
+# Setup test environment variables
+setup_test_env() {
+  export PROJECT_ROOT="$(get_project_root)"
+  export BUNDLE_GEMFILE="${PROJECT_ROOT}/Gemfile"
+
+  # Set database URLs for tests
+  export DATABASE_URL="$(get_postgres_url)"
+  export REDIS_URL="$(get_redis_url)"
+  export DB_CONN_STRING="$DATABASE_URL"
+
+  # Test directories
+  export TEST_PIDDIR="${BATS_TEST_TMPDIR}/pids"
+}
+
+run_background() {
+  local label="$1"
+  shift
+
+  local log_file="$(bg_output_file "$label")"
+  mkdir -p "$TEST_PIDDIR"
+  (
+      "$@" 2>&1 &
+      echo $! >"${TEST_PIDDIR}/${label}.pid"
+  ) | tee "$log_file" | sed "s/^/${label}: /" &
+}
+
+bg_output_file() {
+    local label="$1"
+
+    echo "${BATS_TEST_TMPDIR}/${label}.log"
+}
+
+# A function that polls a given command until it succeeds or until it runs out
+wait_for() {
+  local timeout="$1"
+  local interval="$2"
+  shift 2
+
+  local elapsed=0
+  while [ "$elapsed" -lt "$timeout" ]; do
+    if "$@" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep "$interval"
+    elapsed=$((elapsed + interval))
+  done
+
+  echo "Timeout after ${timeout}s waiting for: $*" >&2
+  return 1
+}