dynflow 1.9.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a3ed6c515677b9b75a3dc93a46780780a1606569d47afcad07206295042d881
-  data.tar.gz: 5740e1c8fae71a447132d52773e268ec7e02886f3e15ad82a6c636e4faf74dcd
+  metadata.gz: 398d1dc51c8a63beda52acc918da0c570ea91ec8c19090fa2cb7ae704cc4b1a2
+  data.tar.gz: 5f1527042cdb29afb348031ced662fb5ac157c4c00ff14fcac73293cba31026c
 SHA512:
-  metadata.gz: b6727fe2aaefe4f025bc443881e5c31dda3d3158f43879e70b9e09e261ea0940e714c9e7cda5fc817f2b3ba5ebb2da74ac0978590a1472d6b532e6542bf9b1ef
-  data.tar.gz: 7c5743be066fc27bc67956e87676ccbb1c6102a1eb03f3ee6ae0cb059d6779d0714d9446895a3a2c49d3c459c217fea262797748266e1718fdc9bd0ba63d37d4
+  metadata.gz: ec917c5c31642bf23ebaf335804e0a151b7ab0a56f8b5d3a5cac23ab63416149844562248bddb93cff293b9d565e02998a958491cdc4f05c94d65859a4d473ef
+  data.tar.gz: 80202d14de3d386a5b8d1c54a48542135ab511421f71e08ef533c50c43fb28900d3318798668fa34fabdf80feec6cba4b3c11635d6ea2b766308d8bc2fe8a2f4

data/.github/workflows/bats.yml

@@ -0,0 +1,50 @@
+name: Bats Integration Tests
+
+on: [pull_request]
+
+jobs:
+  bats:
+    runs-on: ubuntu-latest
+
+    env:
+      DB: postgresql
+
+    steps:
+    - uses: actions/checkout@v5
+
+    - name: Install bats
+      run: |
+        git clone --depth 1 --branch v1.11.0 https://github.com/bats-core/bats-core.git /tmp/bats-core
+        sudo /tmp/bats-core/install.sh /usr/local
+        bats --version
+
+    - name: Install podman
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install podman
+        podman --version
+
+    - id: ruby_version
+      uses: voxpupuli/ruby-version@v1
+    - id: min_ruby
+      run: echo "version=$(echo '${{ steps.ruby_version.outputs.versions }}' | jq -r '.[-1]')" >> $GITHUB_OUTPUT
+    - name: Setup Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        # Use the minimum supported Ruby version for rubocop (last in descending list)
+        ruby-version: ${{ steps.min_ruby.outputs.version }}
+        bundler-cache: true
+
+    - name: Pull container images
+      run: |
+        podman pull docker.io/library/postgres:15
+        podman pull docker.io/library/redis:7-alpine
+
+    - name: Run bats tests
+      run: bats -x --verbose-run --print-output-on-failure test/bats/
+
+    - name: Cleanup containers (if tests fail)
+      if: always()
+      run: |
+        podman stop dynflow-test-postgres dynflow-test-redis 2>/dev/null || true
+        podman rm -f dynflow-test-postgres dynflow-test-redis 2>/dev/null || true

data/.github/workflows/release.yml

@@ -19,7 +19,7 @@ jobs:
           tagRegex: "v(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
           tagRegexGroup: 1 # Optional. Default is 1.
       - name: Checkout the repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v5
       - name: Generate build files
         run: |
           mkdir -p dist

data/.github/workflows/ruby.yml

@@ -15,16 +15,20 @@ env:
 jobs:
   rubocop:
     runs-on: ubuntu-latest
+    outputs:
+      ruby_version: ${{ steps.ruby_version.outputs.versions }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v5
+      - id: ruby_version
+        uses: voxpupuli/ruby-version@v1
+      - id: min_ruby
+        run: echo "version=$(echo '${{ steps.ruby_version.outputs.versions }}' | jq -r '.[-1]')" >> $GITHUB_OUTPUT
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
-      - name: Setup
-        run: |
-          gem install bundler --version=2.4.22
-          bundle install --jobs=3 --retry=3
+          # Use the minimum supported Ruby version for rubocop (last in descending list)
+          ruby-version: ${{ steps.min_ruby.outputs.version }}
+          bundler-cache: true
       - name: Run rubocop
         run: bundle exec rubocop
 
@@ -34,50 +38,30 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby_version:
-          - 2.7.0
-          - 3.0.0
-          - 3.2.0
-          - 3.4
+        ruby_version: ${{ fromJSON(needs.rubocop.outputs.ruby_version) }}
         concurrent_ruby_ext:
           - 'true'
           - 'false'
         db:
           - postgresql
-          - mysql
           - sqlite3
-        include:
-          - db: postgresql
-            conn_string: postgres://postgres@localhost/travis_ci_test
-          - db: mysql
-            conn_string: mysql2://root@127.0.0.1/travis_ci_test
-          - db: sqlite3
-            conn_string: sqlite:/
         exclude:
-          - db: mysql
-            ruby_version: 3.0.0
-          - db: mysql
-            ruby_version: 3.2.0
-          - db: mysql
-            ruby_version: 3.4
-          - db: mysql
-            concurrent_ruby_ext: 'true'
           - db: sqlite3
-            ruby_version: 3.0.0
+            ruby_version: '3.0'
           - db: sqlite3
-            ruby_version: 3.2.0
+            ruby_version: '3.2'
           - db: sqlite3
-            ruby_version: 3.4
+            ruby_version: '3.4'
           - db: sqlite3
             concurrent_ruby_ext: 'true'
           - db: postgresql
-            ruby_version: 3.0.0
+            ruby_version: '3.0'
             concurrent_ruby_ext: 'true'
           - db: postgresql
-            ruby_version: 3.2.0
+            ruby_version: '3.2'
             concurrent_ruby_ext: 'true'
           - db: postgresql
-            ruby_version: 3.4
+            ruby_version: '3.4'
             concurrent_ruby_ext: 'true'
 
     services:
@@ -86,31 +70,28 @@ jobs:
         ports: ['5432:5432']
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
         env:
-          POSTGRES_DB: travis_ci_test
-      mariadb:
-        image: mariadb:10
-        ports: ['3306:3306']
-        env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
-          MYSQL_DATABASE: travis_ci_test
+          POSTGRES_DB: ci_test
       redis:
         image: redis:latest
         ports: ['6379:6379']
 
     env:
       DB: ${{ matrix.db }}
-      DB_CONN_STRING: ${{ matrix.conn_string }}
       CONCURRENT_RUBY_EXT: "${{ matrix.concurrent_ruby_ext }}"
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v5
     - name: Set up Ruby
-    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
-    # change this to (see https://github.com/ruby/setup-ruby#versioning):
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby_version }}
-    - name: Install dependencies
-      run: .github/install_dependencies.sh
+        bundler-cache: true
     - name: Run tests
       run: bundle exec rake test
+    - name: Upload logs
+      uses: actions/upload-artifact@v4
+      if: ${{ failure() && contains(matrix.task, 'test') }}
+      with:
+        name: logs-${{ env.ARTIFACT_SUFFIX }}
+        path: test.log
+        retention-days: 5

data/.gitignore

@@ -7,3 +7,5 @@ Gemfile.lock
 .idea
 .ruby-version
 examples/remote_executor_db.sqlite
+/Gemfile.local.rb
+/test.log

data/.rubocop.yml

@@ -33,3 +33,6 @@ Metrics/ModuleLength:
 
 Style/LambdaCall:
   Enabled: false
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false

data/Dockerfile

@@ -6,4 +6,4 @@ ADD Gemfile /data/
 ADD dynflow.gemspec /data/
 ADD lib/dynflow/version.rb /data/lib/dynflow/version.rb
 WORKDIR /data
-RUN bundle install --without mysql
+RUN bundle install

data/Gemfile

@@ -4,11 +4,11 @@ source 'https://rubygems.org'
 
 gemspec
 
-group :concurrent_ruby_ext do
+group :concurrent_ruby_ext, optional: ENV.key?('CI') && ENV['CONCURRENT_RUBY_EXT'] != 'true' do
   gem 'concurrent-ruby-ext', '~> 1.1.3'
 end
 
-group :pry do
+group :pry, optional: ENV.key?('CI') do
   gem 'pry'
   gem 'pry-byebug'
 end
@@ -18,14 +18,10 @@ group :sidekiq do
   gem 'sidekiq'
 end
 
-group :postgresql do
+group :postgresql, optional: ENV.key?('CI') && ENV['DB'] != 'postgresql' do
   gem "pg"
 end
 
-group :mysql do
-  gem "mysql2"
-end
-
 group :lint do
   gem 'theforeman-rubocop', '~> 0.0.4'
 end
@@ -37,9 +33,12 @@ end
 group :rails do
   gem 'daemons'
   gem 'logging'
-  gem 'rails', '>= 4.2.9', '< 7'
+  gem 'rails', '>= 7', '< 8'
 end
 
 group :telemetry do
   gem 'statsd-instrument'
 end
+
+local_gemfile = File.join(File.dirname(__FILE__), 'Gemfile.local.rb')
+self.instance_eval(Bundler.read_file(local_gemfile)) if File.exist?(local_gemfile)

data/README.md

@@ -126,10 +126,10 @@ The Anatomy of Action Class
 # every action needs to inherit from Dynflow::Action
 class Action < Dynflow::Action
 
-  # OPTIONAL: the input format for the execution phase of this action
-  # (https://github.com/iNecas/apipie-params for more details.
-  # Validations can be performed against this description (turned off
-  # for now)
+  # OPTIONAL: the input format for the execution phase of this action.
+  # This is purely documentation - the block is never evaluated and
+  # serves only as a reference for developers implementing actions.
+  # Input validation is not performed.
   input_format do
     param :id, Integer
     param :name, String

data/doc/pages/source/documentation/index.md

@@ -203,12 +203,12 @@ class AnAction < Dynflow::Action
 end
 ```
 
-This might me quite handy especially in combination with
+This might be quite handy especially in combination with
 [subscriptions](#subscriptions) functionality.
 
-The format follows [apipie-params](https://github.com/iNecas/apipie-params) for more details.
-Validations of input/output could be performed against this description but it's not turned on
-by default. (It needs to be revisited and updated to be fully functional.)
+These format definitions are purely for documentation purposes - the blocks are never
+evaluated and serve only as a reference for developers implementing actions.
+Input/output validation is not performed.
 
 {% endinfo_block %}
 

data/dynflow.gemspec

@@ -14,14 +14,13 @@ Gem::Specification.new do |s|
   s.description = "Ruby workflow/orchestration engine"
   s.license     = "MIT"
 
-  s.files         = `git ls-files`.split("\n")
+  s.files         = `git ls-files`.split("\n").reject { |file| file == '.packit.yaml' }
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.require_paths = ["lib"]
 
-  s.required_ruby_version = '>= 2.7.0'
+  s.required_ruby_version = '>= 3.0.0'
 
   s.add_dependency "algebrick", '~> 0.7.0'
-  s.add_dependency "apipie-params"
   s.add_dependency "concurrent-ruby", '~> 1.1.3'
   s.add_dependency "concurrent-ruby-edge", '~> 0.6.0'
   s.add_dependency "csv", "~> 3.1"

data/examples/example_helper.rb

@@ -48,7 +48,7 @@ class ExampleHelper
     end
 
     def logger_adapter
-      Dynflow::LoggerAdapters::Simple.new $stderr, Logger::FATAL
+      Dynflow::LoggerAdapters::Simple.new $stdout, Logger::DEBUG
     end
 
     def run_web_console(world = ExampleHelper.world)

data/examples/execution_plan_chaining.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative 'example_helper'
+
+class DelayedAction < Dynflow::Action
+  def plan(should_fail = false)
+    plan_self :should_fail => should_fail
+  end
+
+  def run
+    sleep 5
+    raise "Controlled failure" if input[:should_fail]
+  end
+
+  def rescue_strategy
+    Dynflow::Action::Rescue::Fail
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  world = ExampleHelper.create_world do |config|
+    config.auto_rescue = true
+  end
+  world.action_logger.level = 1
+  world.logger.level = 0
+
+  plan1 = world.trigger(DelayedAction)
+  plan2 = world.chain(plan1.execution_plan_id, DelayedAction)
+  plan3 = world.chain(plan2.execution_plan_id, DelayedAction)
+  plan4 = world.chain(plan2.execution_plan_id, DelayedAction)
+
+  plan5 = world.trigger(DelayedAction, true)
+  plan6 = world.chain(plan5.execution_plan_id, DelayedAction)
+
+  puts <<-MSG.gsub(/^.*\|/, '')
+    |
+    |  Execution Plan Chaining example
+    |  ========================
+    |
+    |  This example shows the execution plan chaining functionality of Dynflow, which allows execution plans to wait until another execution plan finishes.
+    |
+    |  Execution plans:
+    |    #{plan1.id} runs immediately and should run successfully.
+    |    #{plan2.id} is delayed and should run once #{plan1.id} finishes.
+    |    #{plan3.id} and #{plan4.id} are delayed and should run once #{plan2.id} finishes.
+    |
+    |    #{plan5.id} runs immediately and is expected to fail.
+    |    #{plan6.id} should not run at all as its prerequisite failed.
+    |
+    |  Visit #{ExampleHelper::DYNFLOW_URL} to see their status.
+    |
+  MSG
+
+  ExampleHelper.run_web_console(world)
+end

data/examples/remote_executor.rb

@@ -117,17 +117,14 @@ class RemoteExecutorExample
       Proc.new { |world| Dynflow::Connectors::Database.new(world) }
     end
 
-    def run_client
+    def run_client(count)
       world = ExampleHelper.create_world do |config|
         config.persistence_adapter = persistence_adapter
         config.executor            = false
         config.connector           = connector
       end
 
-      world.trigger(OrchestrateEvented::CreateInfrastructure)
-      world.trigger(OrchestrateEvented::CreateInfrastructure, true)
-
-      loop do
+      (count || 1000).times do
         start_time = Time.now
         world.trigger(SampleAction).finished.wait
         finished_in = Time.now - start_time
@@ -150,13 +147,13 @@ if $0 == __FILE__
   when 'server'
     puts <<~MSG
       The server is starting…. You can send the work to it by running:
-      
+
          #{$0} client
-      
+
     MSG
     RemoteExecutorExample.run_server
   when 'client'
-    RemoteExecutorExample.run_client
+    RemoteExecutorExample.run_client(ARGV[1]&.to_i)
   else
     puts "Unknown command #{comment}"
     exit 1

data/lib/dynflow/action/format.rb

@@ -1,44 +1,15 @@
 # frozen_string_literal: true
 
 module Dynflow
-  # Input/output format validation logic calling
-  # input_format/output_format with block acts as a setter for
-  # specifying the format. Without a block it acts as a getter
   module Action::Format
-    # we don't evaluate tbe block immediatelly, but postpone it till all the
-    # action classes are loaded, because we can use them to reference output format
     def input_format(&block)
-      case
-      when block && !@input_format_block
-        @input_format_block = block
-      when !block && @input_format_block
-        return @input_format ||= Apipie::Params::Description.define(&@input_format_block)
-      when block && @input_format_block
-        raise "The input_format has already been defined in #{self.class}"
-      when !block && !@input_format_block
-        if superclass.respond_to? :input_format
-          superclass.input_format
-        else
-          raise "The input_format has not been defined yet in #{self.class}"
-        end
-      end
+      # Format definitions are not validated
+      # This method is kept for backward compatibility but does nothing
     end
 
     def output_format(&block)
-      case
-      when block && !@output_format_block
-        @output_format_block = block
-      when !block && @output_format_block
-        return @output_format ||= Apipie::Params::Description.define(&@output_format_block)
-      when block && @output_format_block
-        raise "The output_format has already been defined in #{self.class}"
-      when !block && !@output_format_block
-        if superclass.respond_to? :output_format
-          superclass.output_format
-        else
-          raise "The output_format has not been defined yet in #{self.class}"
-        end
-      end
+      # Format definitions are not validated
+      # This method is kept for backward compatibility but does nothing
     end
   end
 end

data/lib/dynflow/debug/telemetry/persistence.rb

@@ -19,7 +19,7 @@ module Dynflow
           :load_execution_plan,
           :save_execution_plan,
           :find_old_execution_plans,
-          :find_past_delayed_plans,
+          :find_ready_delayed_plans,
           :delete_delayed_plans,
           :save_delayed_plan,
           :set_delayed_plan_frozen,

data/lib/dynflow/delayed_executors/abstract_core.rb

@@ -32,7 +32,7 @@ module Dynflow
 
       def delayed_execution_plans(time)
         with_error_handling([]) do
-          world.persistence.find_past_delayed_plans(time)
+          world.persistence.find_ready_delayed_plans(time)
         end
       end
 

data/lib/dynflow/delayed_plan.rb

@@ -31,6 +31,12 @@ module Dynflow
       error("Execution plan could not be started before set time (#{@start_before})", 'timeout')
     end
 
+    def failed_dependencies(uuids)
+      bullets = uuids.map { |u| "- #{u}" }.join("\n")
+      msg = "Execution plan could not be started because some of its prerequisite execution plans failed:\n#{bullets}"
+      error(msg, 'failed-dependency')
+    end
+
     def error(message, history_entry = nil)
       execution_plan.root_plan_step.state = :error
       execution_plan.root_plan_step.error = ::Dynflow::ExecutionPlan::Steps::Error.new(message)

data/lib/dynflow/director.rb

@@ -114,7 +114,15 @@ module Dynflow
         plan = world.persistence.load_delayed_plan(execution_plan_id)
         return if plan.nil? || plan.execution_plan.state != :scheduled
 
-        if !plan.start_before.nil? && plan.start_before < Time.now.utc()
+        if plan.start_before.nil?
+          blocker_ids = world.persistence.find_execution_plan_dependencies(execution_plan_id)
+          statuses = world.persistence.find_execution_plan_statuses({ filters: { uuid: blocker_ids } })
+          failed = statuses.select { |_uuid, status| status[:state] == 'stopped' && status[:result] == 'error' }
+          if failed.any?
+            plan.failed_dependencies(failed.keys)
+            return
+          end
+        elsif plan.start_before < Time.now.utc()
           plan.timeout
           return
         end

data/lib/dynflow/executors/sidekiq/core.rb

@@ -13,7 +13,7 @@ Sidekiq.configure_server do |config|
   config[:semi_reliable_fetch] = true
   Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
 end
-::Sidekiq.strict_args!(false)
+::Sidekiq.strict_args!(false) if ::Sidekiq.respond_to?(:strict_args!)
 
 module Dynflow
   module Executors

data/lib/dynflow/executors/sidekiq/redis_locking.rb

@@ -41,9 +41,7 @@ module Dynflow
         def wait_for_orchestrator_lock
           mode = nil
           loop do
-            active = ::Sidekiq.redis do |conn|
-              conn.set(REDIS_LOCK_KEY, @world.id, :ex => REDIS_LOCK_TTL, :nx => true)
-            end
+            active = try_acquire_orchestrator_lock
             break if active
             if mode.nil?
               mode = :passive
@@ -54,6 +52,15 @@ module Dynflow
           @logger.info('Acquired orchestrator lock, entering active mode.')
         end
 
+        def try_acquire_orchestrator_lock
+          ::Sidekiq.redis do |conn|
+            conn.set(REDIS_LOCK_KEY, @world.id, :ex => REDIS_LOCK_TTL, :nx => true)
+          end
+        rescue ::Redis::BaseError => e
+          @logger.error("Could not acquire orchestrator lock: #{e}")
+          nil
+        end
+
         def reacquire_orchestrator_lock
           case ::Sidekiq.redis { |conn| conn.eval REACQUIRE_SCRIPT, [REDIS_LOCK_KEY], [@world.id] }
           when ACQUIRE_MISSING

data/lib/dynflow/extensions/msgpack.rb

@@ -16,6 +16,10 @@ module Dynflow
       ::MessagePack::DefaultFactory.register_type(0x00, Time, packer: MessagePack::Time::Packer, unpacker: MessagePack::Time::Unpacker)
 
       begin
+        # time_with_zone added a deprecation warning in 7.1.0 which we need to account for
+        # it was removed again in 7.2.0
+        require 'active_support/deprecation'
+        require 'active_support/deprecator'
         require 'active_support/time_with_zone'
         unpacker = ->(payload) do
           tv = MessagePack::Timestamp.from_msgpack_ext(payload)

data/lib/dynflow/persistence.rb

@@ -101,8 +101,16 @@ module Dynflow
       end
     end
 
-    def find_past_delayed_plans(time)
-      adapter.find_past_delayed_plans(time).map do |plan|
+    def find_execution_plan_dependencies(execution_plan_id)
+      adapter.find_execution_plan_dependencies(execution_plan_id)
+    end
+
+    def find_blocked_execution_plans(execution_plan_id)
+      adapter.find_blocked_execution_plans(execution_plan_id)
+    end
+
+    def find_ready_delayed_plans(time)
+      adapter.find_ready_delayed_plans(time).map do |plan|
         DelayedPlan.new_from_hash(@world, plan)
       end
     end
@@ -163,5 +171,9 @@ module Dynflow
     def prune_undeliverable_envelopes
       adapter.prune_undeliverable_envelopes
     end
+
+    def chain_execution_plan(first, second)
+      adapter.chain_execution_plan(first, second)
+    end
   end
 end

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -72,7 +72,15 @@ module Dynflow
         raise NotImplementedError
       end
 
-      def find_past_delayed_plans(options = {})
+      def find_execution_plan_dependencies(execution_plan_id)
+        raise NotImplementedError
+      end
+
+      def find_blocked_execution_plans(execution_plan_id)
+        raise NotImplementedError
+      end
+
+      def find_ready_delayed_plans(options = {})
         raise NotImplementedError
       end