dynflow 1.9.3 → 2.0.0

data/test/bats/helpers/containers.bash

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# Container helper functions for bats tests
+
+# Default container names
+POSTGRES_CONTAINER_NAME="${POSTGRES_CONTAINER_NAME:-dynflow-test-postgres}"
+REDIS_CONTAINER_NAME="${REDIS_CONTAINER_NAME:-dynflow-test-redis}"
+
+# Default ports
+POSTGRES_PORT="${POSTGRES_PORT:-15432}"
+REDIS_PORT="${REDIS_PORT:-16379}"
+
+# Database credentials
+POSTGRES_USER="${POSTGRES_USER:-dynflow_test}"
+POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-dynflow_test_pass}"
+POSTGRES_DB="${POSTGRES_DB:-dynflow_test}"
+
+# Container images
+POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres:15}"
+REDIS_IMAGE="${REDIS_IMAGE:-docker.io/library/redis:7-alpine}"
+
+# Start PostgreSQL container
+start_postgres() {
+  echo "Starting PostgreSQL container: ${POSTGRES_CONTAINER_NAME}" >&2
+
+  podman run -d \
+    --name "${POSTGRES_CONTAINER_NAME}" \
+    -e POSTGRES_USER="${POSTGRES_USER}" \
+    -e POSTGRES_PASSWORD="${POSTGRES_PASSWORD}" \
+    -e POSTGRES_DB="${POSTGRES_DB}" \
+    -p "${POSTGRES_PORT}:5432" \
+    "${POSTGRES_IMAGE}" \
+    postgres -c fsync=off -c synchronous_commit=off -c full_page_writes=off
+
+  # Wait for PostgreSQL to be ready
+  echo "Waiting for PostgreSQL to be ready..." >&2
+  local max_attempts=30
+  local attempt=0
+
+  while [ $attempt -lt $max_attempts ]; do
+    if podman exec "${POSTGRES_CONTAINER_NAME}" pg_isready -U "${POSTGRES_USER}" > /dev/null 2>&1; then
+      echo "PostgreSQL is ready" >&2
+      return 0
+    fi
+    attempt=$((attempt + 1))
+    sleep 1
+  done
+
+  echo "ERROR: PostgreSQL failed to start within ${max_attempts} seconds" >&2
+  return 1
+}
+
+stop_container() {
+  local container="$1"
+  local with_volumes="$2"
+
+  echo "Stopping container: ${container}" >&2
+  if podman ps -a --format "{{.Names}}" | grep -q "^${container}$"; then
+    podman stop -t 2 "${container}" > /dev/null 2>&1 || true
+    if [ "$with_volumes" = "1" ]; then
+        podman rm -v -f "${container}" > /dev/null 2>&1 || true
+    else
+        podman rm -f "${container}" > /dev/null 2>&1 || true
+    fi
+  fi
+}
+
+# Stop PostgreSQL container
+stop_postgres() {
+  stop_container "$POSTGRES_CONTAINER_NAME" "$1"
+}
+
+# Start Redis container
+start_redis() {
+  echo "Starting Redis container: ${REDIS_CONTAINER_NAME}" >&2
+
+  podman run -d \
+    --name "${REDIS_CONTAINER_NAME}" \
+    -p "${REDIS_PORT}:6379" \
+    "${REDIS_IMAGE}"
+
+  # Wait for Redis to be ready
+  echo "Waiting for Redis to be ready..." >&2
+  local max_attempts=30
+  local attempt=0
+
+  while [ $attempt -lt $max_attempts ]; do
+    if podman exec "${REDIS_CONTAINER_NAME}" redis-cli ping > /dev/null 2>&1; then
+      echo "Redis is ready" >&2
+      return 0
+    fi
+    attempt=$((attempt + 1))
+    sleep 1
+  done
+
+  echo "ERROR: Redis failed to start within ${max_attempts} seconds" >&2
+  return 1
+}
+
+# Stop Redis container
+stop_redis() {
+  stop_container "$REDIS_CONTAINER_NAME" "$1"
+}
+
+# Check if PostgreSQL container is running
+is_postgres_running() {
+  podman ps --format "{{.Names}}" | grep -q "^${POSTGRES_CONTAINER_NAME}$"
+}
+
+# Check if Redis container is running
+is_redis_running() {
+  podman ps --format "{{.Names}}" | grep -q "^${REDIS_CONTAINER_NAME}$"
+}
+
+# Get PostgreSQL connection string
+get_postgres_url() {
+  echo "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${POSTGRES_PORT}/${POSTGRES_DB}"
+}
+
+# Get Redis URL
+get_redis_url() {
+  echo "redis://localhost:${REDIS_PORT}/0"
+}
+
+# Execute SQL in PostgreSQL container
+exec_sql() {
+  local sql="$1"
+  podman exec "${POSTGRES_CONTAINER_NAME}" \
+    psql -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -c "${sql}"
+}
+
+# Execute Redis command
+exec_redis() {
+  podman exec "${REDIS_CONTAINER_NAME}" redis-cli "$@"
+}
+
+# Clean up all test containers
+cleanup_containers() {
+  stop_postgres 1
+  stop_redis 1
+}
+
+# Start all test containers
+start_containers() {
+  start_postgres || return 1
+  start_redis || return 1
+}

data/test/bats/setup_suite.bash

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Suite-level setup - runs once before all tests
+
+# Load container helpers
+source "$(dirname "${BASH_SOURCE[0]}")/helpers/containers.bash"
+source "$(dirname "${BASH_SOURCE[0]}")/helpers/common.bash"
+
+# This function runs once before all tests in the suite
+setup_suite() {
+  echo "=== Setting up bats test suite ===" >&2
+
+  # Verify podman is available
+  if ! command -v podman &> /dev/null; then
+    echo "ERROR: podman is not installed or not in PATH" >&2
+    exit 1
+  fi
+
+  # Check if bundle is available
+  PROJECT_ROOT="$(get_project_root)"
+  if ! command -v bundle &> /dev/null; then
+    echo "WARNING: bundler is not installed" >&2
+  else
+    # Install dependencies if needed
+    echo "Checking bundle dependencies..." >&2
+    cd "${PROJECT_ROOT}" && bundle check > /dev/null 2>&1 || bundle install
+  fi
+
+  # Pull container images if not already present
+  echo "Checking container images..." >&2
+
+  if ! podman image exists "${POSTGRES_IMAGE}"; then
+    echo "Pulling PostgreSQL image: ${POSTGRES_IMAGE}" >&2
+    podman pull "${POSTGRES_IMAGE}"
+  fi
+
+  if ! podman image exists "${REDIS_IMAGE}"; then
+    echo "Pulling Redis image: ${REDIS_IMAGE}" >&2
+    podman pull "${REDIS_IMAGE}"
+  fi
+
+  # Clean up any existing test containers from previous runs
+  echo "Cleaning up any existing test containers..." >&2
+  cleanup_containers
+
+  echo "=== Test suite setup complete ===" >&2
+}

data/test/bats/sidekiq-orchestrator.bats

@@ -0,0 +1,178 @@
+#!/usr/bin/env bats
+# Example bats test file for Dynflow
+
+# Load helper functions
+load helpers/containers
+load helpers/common
+
+# Setup runs before each test
+setup() {
+  # Setup environment variables
+  setup_test_env
+
+  # Ensure containers are running
+  is_postgres_running && stop_postgres
+  start_postgres
+  is_redis_running && stop_redis
+  start_redis
+}
+
+# Teardown runs after each test
+teardown() {
+  (
+    cd "$TEST_PIDDIR" || return 1
+    shopt -s nullglob
+    for pidfile in * ; do
+        kill -9 "$(cat "$pidfile")"
+    done
+  )
+  cleanup_containers 1
+}
+
+@test "sanity" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -c 1 -r ./examples/remote_executor.rb -q dynflow_orchestrator
+  wait_for 5 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'w1' bundle exec sidekiq -r ./examples/remote_executor.rb -q default
+  wait_for 5 1 grep -P 'class=Dynflow::Executors::Sidekiq::WorkerJobs::DrainMarker.*INFO: done' "$(bg_output_file w1)"
+
+  timeout 10 bundle exec ruby examples/remote_executor.rb client 1
+  wait_for 1 1 grep -P 'dynflow: ExecutionPlan.*running >>.*stopped' "$(bg_output_file o1)"
+}
+
+@test "only one orchestrator can be active at a time" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'o2' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Orchestrator lock already taken, entering passive mode.' "$(bg_output_file o2)"
+}
+
+@test "multiple orchestrators can be active with multiple redis dbs" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'w1' bundle exec sidekiq -r ./examples/remote_executor.rb -q default
+
+  export REDIS_URL=${REDIS_URL%/0}/1
+  run_background 'o2' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o2)"
+
+  run_background 'w2' bundle exec sidekiq -r ./examples/remote_executor.rb -q default
+
+  # The client performs a round robin between the available executors
+  # This should lead to each orchestrator handling one execution plan
+  timeout 60 bundle exec ruby examples/remote_executor.rb client 2
+  wait_for 1 1 grep -P 'dynflow: ExecutionPlan.*running >>.*stopped' "$(bg_output_file o1)"
+  wait_for 1 1 grep -P 'dynflow: ExecutionPlan.*running >>.*stopped' "$(bg_output_file o2)"
+}
+
+@test "orchestrators do fail over" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'o2' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Orchestrator lock already taken, entering passive mode.' "$(bg_output_file o2)"
+
+  kill -15 "$(cat "$TEST_PIDDIR/o1.pid")"
+  wait_for 120 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o2)"
+}
+
+@test "active orchestrator exits when pg goes away for good" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'w1' bundle exec sidekiq -r ./examples/remote_executor.rb -q default
+  wait_for 5 1 grep 'dynflow: Finished performing validity checks' "$(bg_output_file o1)"
+
+  podman stop "$POSTGRES_CONTAINER_NAME"
+  wait_for 60 1 grep 'dynflow: World terminated, exiting.' "$(bg_output_file o1)"
+}
+
+@test "active orchestrator can withstand temporary pg connection drop" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'w1' bundle exec sidekiq -r ./examples/remote_executor.rb -q default
+  wait_for 5 1 grep 'dynflow: Finished performing validity checks' "$(bg_output_file o1)"
+
+  podman stop "$POSTGRES_CONTAINER_NAME"
+  wait_for 30 1 grep 'dynflow: Persistence retry no. 1' "$(bg_output_file o1)"
+  podman start "$POSTGRES_CONTAINER_NAME"
+  wait_for 30 1 grep 'dynflow: Executor heartbeat' "$(bg_output_file o1)"
+
+  timeout 30 bundle exec ruby examples/remote_executor.rb client 1
+  wait_for 1 1 grep -P 'dynflow: ExecutionPlan.*running >>.*stopped' "$(bg_output_file o1)"
+}
+
+@test "active orchestrator can survive a brief redis connection drop" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'w1' bundle exec sidekiq -r ./examples/remote_executor.rb -q default
+  wait_for 5 1 grep 'dynflow: Finished performing validity checks' "$(bg_output_file o1)"
+
+  stop_redis
+  wait_for 30 1 grep 'Error connecting to Redis' "$(bg_output_file o1)"
+  start_redis
+
+  timeout 10 bundle exec ruby examples/remote_executor.rb client 1
+  wait_for 1 1 grep -P 'dynflow: ExecutionPlan.*running >>.*stopped' "$(bg_output_file o1)"
+}
+
+@test "active orchestrator can survive a longer redis connection drop" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'w1' bundle exec sidekiq -r ./examples/remote_executor.rb -q default
+  wait_for 5 1 grep 'dynflow: Finished performing validity checks' "$(bg_output_file o1)"
+
+  stop_redis 1
+  wait_for 30 1 grep 'Error connecting to Redis' "$(bg_output_file o1)"
+  start_redis
+
+  wait_for 30 1 grep 'The orchestrator lock was lost, reacquired' "$(bg_output_file o1)"
+
+  timeout 10 bundle exec ruby examples/remote_executor.rb client 1
+  wait_for 1 1 grep -P 'dynflow: ExecutionPlan.*running >>.*stopped' "$(bg_output_file o1)"
+}
+
+@test "orchestrators can fail over if active one goes away during downtime" {
+  cd "$(get_project_root)"
+
+  run_background 'o1' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o1)"
+
+  run_background 'o2' bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  wait_for 30 1 grep 'dynflow: Orchestrator lock already taken, entering passive mode.' "$(bg_output_file o2)"
+
+  run_background 'w1' bundle exec sidekiq -r ./examples/remote_executor.rb -q default
+  wait_for 5 1 grep 'dynflow: Finished performing validity checks' "$(bg_output_file o1)"
+
+  stop_redis 1
+  wait_for 30 1 grep 'Error connecting to Redis' "$(bg_output_file o1)"
+  kill -15 "$(cat "$TEST_PIDDIR/o1.pid")"
+  start_redis
+
+  wait_for 120 1 grep 'dynflow: Acquired orchestrator lock, entering active mode.' "$(bg_output_file o2)"
+  wait_for 120 1 grep 'dynflow: Finished performing validity checks' "$(bg_output_file o2)"
+
+  timeout 10 bundle exec ruby examples/remote_executor.rb client 1
+  wait_for 1 1 grep -P 'dynflow: ExecutionPlan.*running >>.*stopped' "$(bg_output_file o2)"
+}

data/test/bats/teardown_suite.bash

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Suite-level teardown - runs once after all tests
+
+# Load container helpers
+source "$(dirname "${BASH_SOURCE[0]}")/helpers/containers.bash"
+
+# This function runs once after all tests in the suite
+teardown_suite() {
+  echo "=== Tearing down bats test suite ===" >&2
+
+  # Clean up all test containers
+  echo "Cleaning up test containers..." >&2
+  cleanup_containers
+
+  echo "=== Test suite teardown complete ===" >&2
+}

data/test/concurrency_control_test.rb

@@ -54,7 +54,6 @@ module Dynflow
         def run(event = nil)
           unless output[:slept]
             output[:slept] = true
-            puts "SLEEPING" if input[:should_sleep]
             suspend { |suspended| world.clock.ping(suspended, 100, [:run]) } if input[:should_sleep]
           end
         end

data/test/daemon_test.rb

@@ -5,8 +5,24 @@ require 'active_support'
 require 'mocha/minitest'
 require 'logging'
 require 'dynflow/testing'
+require 'ostruct'
 require_relative '../lib/dynflow/rails'
 
+class StdIOWrapper
+  def initialize(logger, error = false)
+    @logger = logger
+    @error = error
+  end
+
+  def puts(msg = nil)
+    if @error
+      @logger.error(msg)
+    else
+      @logger.info(msg)
+    end
+  end
+end
+
 class DaemonTest < ActiveSupport::TestCase
   setup do
     @dynflow_memory_watcher = mock('memory_watcher')
@@ -15,6 +31,9 @@ class DaemonTest < ActiveSupport::TestCase
       @dynflow_memory_watcher,
       @daemons
     )
+    logger = WorldFactory.logger_adapter.logger
+    @daemon.stubs(:stdout).returns(StdIOWrapper.new(logger, false))
+    @daemon.stubs(:stderr).returns(StdIOWrapper.new(logger, true))
     @world_class = mock('dummy world factory')
     @dummy_world = ::Dynflow::Testing::DummyWorld.new
     @dummy_world.stubs(:id => '123')
@@ -27,9 +46,9 @@ class DaemonTest < ActiveSupport::TestCase
       @world_class,
       ::Dynflow::Rails::Configuration.new
     )
-    ::Rails.stubs(:application).returns(OpenStruct.new(:dynflow => @dynflow))
+    ::Rails.stubs(:application).returns(::OpenStruct.new(:dynflow => @dynflow))
     ::Rails.stubs(:root).returns('support/rails')
-    ::Rails.stubs(:logger).returns(Logging.logger(STDOUT))
+    ::Rails.stubs(:logger).returns(logger)
     @dynflow.require!
     @dynflow.config.stubs(:increase_db_pool_size? => false)
     @daemon.stubs(:sleep).returns(true) # don't pause the execution

data/test/extensions_test.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
 require_relative 'test_helper'
-require 'active_support/time'
+require 'active_support/all'
 
 module Dynflow
   module ExtensionsTest
     describe 'msgpack extensions' do
       before do
-        Thread.current[:time_zone] = ActiveSupport::TimeZone['Europe/Prague']
+        Time.zone = ActiveSupport::TimeZone['Europe/Prague']
       end
-      after { Thread.current[:time_zone] = nil }
+      after { Time.zone = nil }
 
       it 'allows {de,}serializing Time' do
         time = Time.now

data/test/future_execution_test.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'test_helper'
+require 'multi_json'
 
 module Dynflow
   module FutureExecutionTest
@@ -75,7 +76,7 @@ module Dynflow
         it 'finds delayed plans' do
           @start_at = Time.now.utc - 100
           delayed_plan
-          past_delayed_plans = world.persistence.find_past_delayed_plans(@start_at + 10)
+          past_delayed_plans = world.persistence.find_ready_delayed_plans(@start_at + 10)
           _(past_delayed_plans.length).must_equal 1
           _(past_delayed_plans.first.execution_plan_uuid).must_equal execution_plan.id
         end
@@ -112,8 +113,8 @@ module Dynflow
 
         it 'checks for delayed plans in regular intervals' do
           start_time = klok.current_time
-          persistence.expect(:find_past_delayed_plans, [], [start_time])
-          persistence.expect(:find_past_delayed_plans, [], [start_time + options[:poll_interval]])
+          persistence.expect(:find_ready_delayed_plans, [], [start_time])
+          persistence.expect(:find_ready_delayed_plans, [], [start_time + options[:poll_interval]])
           dummy_world.stub :persistence, persistence do
             _(klok.pending_pings.length).must_equal 0
             delayed_executor.start.wait
@@ -189,6 +190,152 @@ module Dynflow
           _(serializer.args).must_equal args
         end
       end
+
+      describe 'execution plan chaining' do
+        let(:world) do
+          WorldFactory.create_world { |config| config.auto_rescue = true }
+        end
+
+        before do
+          @preexisting = world.persistence.find_ready_delayed_plans(Time.now).map(&:execution_plan_uuid)
+        end
+
+        it 'chains two execution plans' do
+          plan1 = world.plan(Support::DummyExample::Dummy)
+          plan2 = world.chain(plan1.id, Support::DummyExample::Dummy)
+
+          Concurrent::Promises.resolvable_future.tap do |promise|
+            world.execute(plan1.id, promise)
+          end.wait
+
+          plan1 = world.persistence.load_execution_plan(plan1.id)
+          _(plan1.state).must_equal :stopped
+          ready = world.persistence.find_ready_delayed_plans(Time.now).reject { |p| @preexisting.include? p.execution_plan_uuid }
+          _(ready.count).must_equal 1
+          _(ready.first.execution_plan_uuid).must_equal plan2.execution_plan_id
+        end
+
+        it 'chains onto multiple execution plans and waits for all to finish' do
+          plan1 = world.plan(Support::DummyExample::Dummy)
+          plan2 = world.plan(Support::DummyExample::Dummy)
+          plan3 = world.chain([plan2.id, plan1.id], Support::DummyExample::Dummy)
+
+          # Execute and complete plan1
+          Concurrent::Promises.resolvable_future.tap do |promise|
+            world.execute(plan1.id, promise)
+          end.wait
+
+          plan1 = world.persistence.load_execution_plan(plan1.id)
+          _(plan1.state).must_equal :stopped
+
+          # plan3 should still not be ready because plan2 hasn't finished yet
+          ready = world.persistence.find_ready_delayed_plans(Time.now).reject { |p| @preexisting.include? p.execution_plan_uuid }
+          _(ready.count).must_equal 0
+
+          # Execute and complete plan2
+          Concurrent::Promises.resolvable_future.tap do |promise|
+            world.execute(plan2.id, promise)
+          end.wait
+
+          plan2 = world.persistence.load_execution_plan(plan2.id)
+          _(plan2.state).must_equal :stopped
+
+          # Now plan3 should be ready since both plan1 and plan2 are complete
+          ready = world.persistence.find_ready_delayed_plans(Time.now).reject { |p| @preexisting.include? p.execution_plan_uuid }
+          _(ready.count).must_equal 1
+          _(ready.first.execution_plan_uuid).must_equal plan3.execution_plan_id
+        end
+
+        it 'cancels the chained plan if the prerequisite fails' do
+          plan1 = world.plan(Support::DummyExample::FailingDummy)
+          plan2 = world.chain(plan1.id, Support::DummyExample::Dummy)
+
+          Concurrent::Promises.resolvable_future.tap do |promise|
+            world.execute(plan1.id, promise)
+          end.wait
+
+          plan1 = world.persistence.load_execution_plan(plan1.id)
+          _(plan1.state).must_equal :stopped
+          _(plan1.result).must_equal :error
+
+          # plan2 will appear in ready delayed plans
+          ready = world.persistence.find_ready_delayed_plans(Time.now).reject { |p| @preexisting.include? p.execution_plan_uuid }
+          _(ready.map(&:execution_plan_uuid)).must_equal [plan2.execution_plan_id]
+
+          # Process the delayed plan through the director
+          work_item = Dynflow::Director::PlanningWorkItem.new(plan2.execution_plan_id, :default, world.id)
+          work_item.world = world
+          work_item.execute
+
+          # Now plan2 should be stopped with error due to failed dependency
+          plan2 = world.persistence.load_execution_plan(plan2.execution_plan_id)
+          _(plan2.state).must_equal :stopped
+          _(plan2.result).must_equal :error
+          _(plan2.errors.first.message).must_match(/prerequisite execution plans failed/)
+          _(plan2.errors.first.message).must_match(/#{plan1.id}/)
+        end
+
+        it 'cancels the chained plan if at least one prerequisite fails' do
+          plan1 = world.plan(Support::DummyExample::Dummy)
+          plan2 = world.plan(Support::DummyExample::FailingDummy)
+          plan3 = world.chain([plan1.id, plan2.id], Support::DummyExample::Dummy)
+
+          # Execute and complete plan1 successfully
+          Concurrent::Promises.resolvable_future.tap do |promise|
+            world.execute(plan1.id, promise)
+          end.wait
+
+          plan1 = world.persistence.load_execution_plan(plan1.id)
+          _(plan1.state).must_equal :stopped
+          _(plan1.result).must_equal :success
+
+          # plan3 should still not be ready because plan2 hasn't finished yet
+          ready = world.persistence.find_ready_delayed_plans(Time.now).reject { |p| @preexisting.include? p.execution_plan_uuid }
+          _(ready).must_equal []
+
+          # Execute and complete plan2 with failure
+          Concurrent::Promises.resolvable_future.tap do |promise|
+            world.execute(plan2.id, promise)
+          end.wait
+
+          plan2 = world.persistence.load_execution_plan(plan2.id)
+          _(plan2.state).must_equal :stopped
+          _(plan2.result).must_equal :error
+
+          # plan3 will now appear in ready delayed plans even though one prerequisite failed
+          ready = world.persistence.find_ready_delayed_plans(Time.now).reject { |p| @preexisting.include? p.execution_plan_uuid }
+          _(ready.map(&:execution_plan_uuid)).must_equal [plan3.execution_plan_id]
+
+          # Process the delayed plan through the director
+          work_item = Dynflow::Director::PlanningWorkItem.new(plan3.execution_plan_id, :default, world.id)
+          work_item.world = world
+          work_item.execute
+
+          # Now plan3 should be stopped with error due to failed dependency
+          plan3 = world.persistence.load_execution_plan(plan3.execution_plan_id)
+          _(plan3.state).must_equal :stopped
+          _(plan3.result).must_equal :error
+          _(plan3.errors.first.message).must_match(/prerequisite execution plans failed/)
+          _(plan3.errors.first.message).must_match(/#{plan2.id}/)
+        end
+
+        it 'chains runs the chained plan if the prerequisite was halted' do
+          plan1 = world.plan(Support::DummyExample::Dummy)
+          plan2 = world.chain(plan1.id, Support::DummyExample::Dummy)
+
+          world.halt(plan1.id)
+          Concurrent::Promises.resolvable_future.tap do |promise|
+            world.execute(plan1.id, promise)
+          end.wait
+
+          plan1 = world.persistence.load_execution_plan(plan1.id)
+          _(plan1.state).must_equal :stopped
+          _(plan1.result).must_equal :pending
+          ready = world.persistence.find_ready_delayed_plans(Time.now).reject { |p| @preexisting.include? p.execution_plan_uuid }
+          _(ready.count).must_equal 1
+          _(ready.first.execution_plan_uuid).must_equal plan2.execution_plan_id
+        end
+      end
     end
   end
 end