dynflow 1.4.3 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 318bdedfaca8720a62bc3a8bbd7d52b1823e9078ed1b96fecea248a729218335
-  data.tar.gz: 35cfd900ad187608dbefaae809af955a8433a1168929d48ff11e92d164cfc8aa
+  metadata.gz: 5b33bcad7864102ad9f0f3bb654c7990b3037b7b0620b770d5f34f1495402855
+  data.tar.gz: f4089f0cb793384a764f4aa042268c19d174c989c5d348f63e126eac5fb94716
 SHA512:
-  metadata.gz: 729dd7445b1bae4d6874749539c191912e6468cdcc120340121e9258502f8a6adc9fc7c90d043b3d4ffc6e53392910d75565233f29ce1b48448503aa366b643a
-  data.tar.gz: 9246e6fd8a38383a5e5fd0c17d4dea5b4cc173f03b47c912673db8514cab773ae6c845c47583b85e9f8e833208751273888f73284fb4f90b451d2ed56b02998d
+  metadata.gz: c119f0ce0d3605012206b083b829bc85194d676f762d61a394be6257d376c56388efd72f4a77f78445ece78942c5eb561728031e9a703767113eab7780b9d0e3
+  data.tar.gz: 136dafb81d0766e9c1041a9a52896b4cbe0d300482be729900fd9fe8f83e6095bbc084a6d43d0aa9012b436544690d3c2d774d46f0b9c573775b3707daec3428

data/{test/prepare_travis_env.sh → .github/install_dependencies.sh}

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -x
+
 echo "Setting the environment to use ${DB} database"
 
 BUNDLE_CONFIG=.bundle/config
@@ -12,11 +14,9 @@ EOF
 case $DB in
     mysql)
         sed -i 's/:mysql//'g $BUNDLE_CONFIG
-        mysql -e 'create database travis_ci_test;'
     ;;
     postgresql)
         sed -i 's/:postgresql//'g $BUNDLE_CONFIG
-        psql -c 'create database travis_ci_test;' -U postgres
     ;;
     sqlite3)
         # the tests are by default using sqlite3: do nothing

data/.github/workflows/ruby.yml

@@ -0,0 +1,116 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on: [pull_request]
+
+env:
+  TESTOPTS: --verbose
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: Setup
+        run: |
+          gem install bundler
+          bundle install --jobs=3 --retry=3
+      - name: Run rubocop
+        run: bundle exec rubocop
+
+  test:
+    runs-on: ubuntu-latest
+    needs: rubocop
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby_version:
+          - 2.5.0
+          - 2.6.0
+          - 2.7.0
+          - 3.0.0
+        concurrent_ruby_ext:
+          - 'true'
+          - 'false'
+        db:
+          - postgresql
+          - mysql
+          - sqlite3
+        include:
+          - db: postgresql
+            conn_string: postgres://postgres@localhost/travis_ci_test
+          - db: mysql
+            conn_string: mysql2://root@127.0.0.1/travis_ci_test
+          - db: sqlite3
+            conn_string: sqlite:/
+        exclude:
+          - db: mysql
+            ruby_version: 2.5.0
+          - db: mysql
+            ruby_version: 2.6.0
+          - db: mysql
+            ruby_version: 3.0.0
+          - db: mysql
+            concurrent_ruby_ext: 'true'
+          - db: sqlite3
+            ruby_version: 2.5.0
+          - db: sqlite3
+            ruby_version: 2.6.0
+          - db: sqlite3
+            ruby_version: 3.0.0
+          - db: sqlite3
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 2.5.0
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 2.6.0
+            concurrent_ruby_ext: 'true'
+          - db: postgresql
+            ruby_version: 3.0.0
+            concurrent_ruby_ext: 'true'
+
+    services:
+      postgres:
+        image: postgres:12.1
+        ports: ['5432:5432']
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        env:
+          POSTGRES_DB: travis_ci_test
+      mariadb:
+        image: mariadb:10
+        ports: ['3306:3306']
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+          MYSQL_DATABASE: travis_ci_test
+      redis:
+        image: redis:latest
+        ports: ['6379:6379']
+
+    env:
+      DB: ${{ matrix.db }}
+      DB_CONN_STRING: ${{ matrix.conn_string }}
+      CONCURRENT_RUBY_EXT: "${{ matrix.concurrent_ruby_ext }}"
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby_version }}
+    - name: Install dependencies
+      run: .github/install_dependencies.sh
+    - name: Run tests
+      run: bundle exec rake test

data/lib/dynflow.rb

@@ -76,7 +76,7 @@ module Dynflow
   if defined? ::ActiveJob
     require 'dynflow/active_job/queue_adapter'
 
-    class Railtie < Rails::Railtie
+    class Railtie < ::Rails::Railtie
       config.before_initialize do
         ::ActiveJob::QueueAdapters.send(
           :include,

data/lib/dynflow/action.rb

@@ -93,7 +93,8 @@ module Dynflow
       fields! execution_plan_id: String,
               step_id:           Integer,
               event:             Object,
-              time:              type { variants Time, NilClass }
+              time:              type { variants Time, NilClass },
+              optional:          Algebrick::Types::Boolean
     end
 
     def self.constantize(action_name)
@@ -332,9 +333,9 @@ module Dynflow
 
     # Plan an +event+ to be send to the action defined by +action+, what defaults to be self.
     # if +time+ is not passed, event is sent as soon as possible.
-    def plan_event(event, time = nil, execution_plan_id: self.execution_plan_id, step_id: self.run_step_id)
+    def plan_event(event, time = nil, execution_plan_id: self.execution_plan_id, step_id: self.run_step_id, optional: false)
       time = @world.clock.current_time + time if time.is_a?(Numeric)
-      delayed_events << DelayedEvent[execution_plan_id, step_id, event, time]
+      delayed_events << DelayedEvent[execution_plan_id, step_id, event, time, optional]
     end
 
     def delayed_events
@@ -352,15 +353,12 @@ module Dynflow
       @step.state = state
     end
 
+    # If this save returns an integer, it means it was an update. The number
+    # represents the number of updated records. If it is 0, then the step was in
+    # an unexpected state and couldn't be updated
     def save_state(conditions = {})
       phase! Executable
-      # If this save returns an integer, it means it was an update. The number
-      #   represents the number of updated records. If it is 0, then the step
-      #   was in an unexpected state and couldn't be updated, in which case we
-      #   raise an exception and crash hard to prevent the step from being
-      #   executed twice
-      count = @step.save(conditions)
-      raise 'Could not save state' if count.kind_of?(Integer) && !count.positive?
+      @step.save(conditions)
     end
 
     def delay(delay_options, *args)
@@ -536,11 +534,11 @@ module Dynflow
     end
 
     # TODO: This is getting out of hand, refactoring needed
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
     def execute_run(event)
       phase! Run
       @world.logger.debug format('%13s %s:%2d got event %s',
                                  'Step', execution_plan_id, @step.id, event) if event
-      @input = OutputReference.dereference @input, world.persistence
 
       case
       when state == :running
@@ -551,8 +549,19 @@ module Dynflow
           raise 'event can be processed only when in suspended state'
         end
 
+        old_state = self.state
         self.state = :running unless self.state == :skipping
-        save_state(:state => %w(pending error skipping suspended))
+        saved = save_state(:state => %w(pending error skipping suspended))
+        if saved.kind_of?(Integer) && !saved.positive?
+          # The step was already in a state we're trying to transition to, most
+          # likely we were about to execute it for the second time after first
+          # execution was forcefully interrupted.
+          # Set error and return to prevent the step from being executed twice
+          set_error "Could not transition step from #{old_state} to #{self.state}, step already in #{self.state}."
+          return
+        end
+
+        @input = OutputReference.dereference @input, world.persistence
         with_error_handling do
           event = Skip if state == :skipping
 
@@ -573,6 +582,7 @@ module Dynflow
         raise "wrong state #{state} when event:#{event}"
       end
     end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
     def execute_finalize
       phase! Finalize

data/lib/dynflow/action/suspended.rb

@@ -9,14 +9,14 @@ module Dynflow
       @step_id           = action.run_step_id
     end
 
-    def plan_event(event, time, sent = Concurrent::Promises.resolvable_future)
-      @world.plan_event(execution_plan_id, step_id, event, time, sent)
+    def plan_event(event, time, sent = Concurrent::Promises.resolvable_future, optional: false)
+      @world.plan_event(execution_plan_id, step_id, event, time, sent, optional: optional)
     end
 
-    def event(event, sent = Concurrent::Promises.resolvable_future)
+    def event(event, sent = Concurrent::Promises.resolvable_future, optional: false)
       # TODO: deprecate 2 levels backtrace (to know it's called from clock or internaly)
       # remove lib/dynflow/clock.rb ClockReference#ping branch condition on removal.
-      plan_event(event, nil, sent)
+      plan_event(event, nil, sent, optional: optional)
     end
 
     def <<(event = nil)

data/lib/dynflow/action/timeouts.rb

@@ -7,8 +7,8 @@ module Dynflow
       fail("Timeout exceeded.")
     end
 
-    def schedule_timeout(seconds)
-      plan_event(Timeout, seconds)
+    def schedule_timeout(seconds, optional: false)
+      plan_event(Timeout, seconds, optional: optional)
     end
  end
 end

data/lib/dynflow/actor.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 module Dynflow
 
+  FULL_BACKTRACE = %w[1 y yes].include?((ENV['DYNFLOW_FULL_BACKTRACE'] || '').downcase)
+  BACKTRACE_LIMIT = begin
+                      limit = ENV['DYNFLOW_BACKTRACE_LIMIT'].to_i
+                      limit.zero? ? nil : limit
+                    end
+
   module MethodicActor
     def on_message(message)
       method, *args = message
@@ -44,7 +50,11 @@ module Dynflow
       include LogWithFullBacktrace
 
       def on_envelope(envelope)
-        Actor::BacktraceCollector.with_backtrace(envelope.origin_backtrace) { super }
+        if FULL_BACKTRACE
+          Actor::BacktraceCollector.with_backtrace(envelope.origin_backtrace) { super }
+        else
+          super
+        end
       end
     end
 
@@ -83,9 +93,15 @@ module Dynflow
 
         # takes an array of backtrace lines and replaces each chunk
         def filter_backtrace(backtrace)
-          backtrace.map { |line| filter_line(line) }
-            .chunk_while { |l1, l2| l1 == l2}
-            .map(&:first)
+          trace = backtrace.map { |line| filter_line(line) }
+                           .chunk_while { |l1, l2| l1 == l2}
+                           .map(&:first)
+          if BACKTRACE_LIMIT
+            count = trace.count
+            trace = trace.take(BACKTRACE_LIMIT)
+            trace << "[ backtrace omitted #{count - BACKTRACE_LIMIT} lines ]" if trace.count < count
+          end
+          trace
         end
       end
     end

data/lib/dynflow/clock.rb

@@ -114,11 +114,11 @@ module Dynflow
       Time.now
     end
 
-    def ping(who, time, with_what = nil, where = :<<)
+    def ping(who, time, with_what = nil, where = :<<, optional: false)
       Type! time, Time, Numeric
       time  = current_time + time if time.is_a? Numeric
       if who.is_a?(Action::Suspended)
-        who.plan_event(with_what, time)
+        who.plan_event(with_what, time, optional: optional)
       else
         timer = Clock::Timer[who, time, with_what.nil? ? Algebrick::Types::None : Some[Object][with_what], where]
         self.tell([:add_timer, timer])

data/lib/dynflow/connectors/abstract.rb

@@ -25,6 +25,10 @@ module Dynflow
         raise NotImplementedError
       end
 
+      def prune_undeliverable_envelopes(world)
+        raise NotImplementedError
+      end
+
       # we need to pass the world, as the connector can be shared
       # between words: we need to know the one to send the message to
       def receive(world, envelope)

data/lib/dynflow/connectors/database.rb

@@ -172,6 +172,10 @@ module Dynflow
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id, :direction => 'outgoing') }
         @core.ask([:handle_envelope, envelope])
       end
+
+      def prune_undeliverable_envelopes(world)
+        world.persistence.prune_undeliverable_envelopes
+      end
     end
   end
 end

data/lib/dynflow/connectors/direct.rb

@@ -68,6 +68,11 @@ module Dynflow
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id) }
         @core.ask([:handle_envelope, envelope])
       end
+
+      def prune_undeliverable_envelopes(_world)
+        # This is a noop
+        0
+      end
     end
   end
 end

data/lib/dynflow/director.rb

@@ -15,7 +15,8 @@ module Dynflow
               execution_plan_id: String,
               step_id:           Integer,
               event:             Object,
-              result:            Concurrent::Promises::ResolvableFuture
+              result:            Concurrent::Promises::ResolvableFuture,
+              optional:          Algebrick::Types::Boolean
     end
 
     UnprocessableEvent = Class.new(Dynflow::Error)
@@ -163,6 +164,9 @@ module Dynflow
       execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
       if execution_plan_manager
         execution_plan_manager.event(event)
+      elsif event.optional
+        event.result.reject "no manager for #{event.inspect}"
+        []
       else
         raise Dynflow::Error, "no manager for #{event.inspect}"
       end

data/lib/dynflow/director/running_steps_manager.rb

@@ -20,8 +20,8 @@ module Dynflow
       def terminate
         pending_work = @work_items.clear.values.flatten(1)
         pending_work.each do |w|
-          if EventWorkItem === w
-            w.event.result.reject UnprocessableEvent.new("dropping due to termination")
+          finish_event_result(w) do |result|
+            result.reject UnprocessableEvent.new("dropping due to termination")
           end
         end
       end

data/lib/dynflow/dispatcher.rb

@@ -6,7 +6,8 @@ module Dynflow
         fields! execution_plan_id: String,
                 step_id:           Integer,
                 event:             Object,
-                time:              type { variants Time, NilClass }
+                time:              type { variants Time, NilClass },
+                optional:          Algebrick::Types::Boolean
       end
 
       Execution = type do

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -132,11 +132,13 @@ module Dynflow
       end
 
       def dispatch_request(request, client_world_id, request_id)
+        ignore_unknown = false
         executor_id = match request,
                             (on ~Execution do |execution|
                                AnyExecutor
                              end),
                             (on ~Event do |event|
+                               ignore_unknown = event.optional
                                find_executor(event.execution_plan_id)
                              end),
                             (on Ping.(~any, ~any) | Status.(~any, ~any) do |receiver_id, _|
@@ -144,7 +146,11 @@ module Dynflow
                              end)
         envelope = Envelope[request_id, client_world_id, executor_id, request]
         if Dispatcher::UnknownWorld === envelope.receiver_id
-          raise Dynflow::Error, "Could not find an executor for #{envelope}"
+          raise Dynflow::Error, "Could not find an executor for #{envelope}" unless ignore_unknown
+
+          message = "Could not find an executor for optional #{envelope}, discarding."
+          log(Logger::DEBUG, message)
+          return respond(envelope, Failed[message])
         end
         connector.send(envelope).value!
       rescue => e
@@ -252,7 +258,7 @@ module Dynflow
           future.fulfill(true)
         else
           if @ping_cache.executor?(request.receiver_id)
-            future.reject
+            future.reject false
           else
             yield
           end