dynflow 1.4.3 → 1.5.0

data/lib/dynflow/testing/in_thread_executor.rb

@@ -34,8 +34,8 @@ module Dynflow
         @director.work_finished(work_item)
       end
 
-      def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
-        event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future])
+      def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future, optional: false)
+        event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future, optional])
         @director.handle_event(event).each do |work_item|
           @work_items << work_item
         end

data/lib/dynflow/testing/in_thread_world.rb

@@ -58,15 +58,15 @@ module Dynflow
         future.reject e
       end
 
-      def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
-        @executor.event(execution_plan_id, step_id, event, done)
+      def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
+        @executor.event(execution_plan_id, step_id, event, done, optional: optional)
       end
 
-      def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future)
+      def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future, optional: false)
         if time.nil? || time < Time.now
-          event(execution_plan_id, step_id, event, done)
+          event(execution_plan_id, step_id, event, done, optional: optional)
         else
-          clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done], :delayed_event)
+          clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done, optional], :delayed_event)
         end
       end
     end

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.4.3'
+  VERSION = '1.5.0'
 end

data/lib/dynflow/world.rb

@@ -219,12 +219,12 @@ module Dynflow
       publish_request(Dispatcher::Execution[execution_plan_id], done, true)
     end
 
-    def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
-      publish_request(Dispatcher::Event[execution_plan_id, step_id, event], done, false)
+    def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
+      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, nil, optional], done, false)
     end
 
-    def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future)
-      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time], accepted, false)
+    def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future, optional: false)
+      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time, optional], accepted, false)
     end
 
     def ping(world_id, timeout, done = Concurrent::Promises.resolvable_future)
@@ -341,7 +341,7 @@ module Dynflow
         @terminating = Concurrent::Promises.future do
           termination_future.wait(termination_timeout)
         end.on_resolution do
-          @terminated.complete
+          @terminated.resolve
           Thread.new { Kernel.exit } if @exit_on_terminate.true?
         end
       end

data/lib/dynflow/world/invalidation.rb

@@ -28,6 +28,8 @@ module Dynflow
             end
           end
 
+          pruned = persistence.prune_envelopes(world.id)
+          logger.error("Pruned #{pruned} envelopes for invalidated world #{world.id}") unless pruned.zero?
           coordinator.delete_world(world)
         end
       end
@@ -41,7 +43,7 @@ module Dynflow
                   else
                     :stopped
                   end
-          plan.update_state(state)
+          plan.update_state(state) if plan.state != state
 
           coordinator.release(planning_lock)
           execute(plan.id) if plan.state == :planned
@@ -115,6 +117,8 @@ module Dynflow
       def perform_validity_checks
         world_invalidation_result = worlds_validity_check
         locks_validity_check
+        pruned = connector.prune_undeliverable_envelopes(self)
+        logger.error("Pruned #{pruned} undeliverable envelopes") unless pruned.zero?
         world_invalidation_result.values.select { |result| result == :invalidated }.size
       end
 

data/test/dispatcher_test.rb

@@ -49,6 +49,12 @@ module Dynflow
               plan = result.finished.value
               assert_equal('finish', plan.actions.first.output[:event])
             end
+
+            it 'does not error on dispatching an optional event' do
+              request = client_world.event('123', 1, nil, optional: true)
+              request.wait(20)
+              assert_match /Could not find an executor for optional .*, discarding/, request.reason.message
+            end
           end
         end
       end

data/test/flows_test.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+require_relative 'test_helper'
+require 'mocha/minitest'
+
+module Dynflow
+  describe 'flow' do
+
+    class TestRegistry < Flows::Registry
+      class << self
+        def reset!
+          @serialization_map = {}
+        end
+      end
+    end
+
+    after do
+      TestRegistry.reset!
+    end
+
+    describe "registry" do
+      it "allows registering values" do
+        TestRegistry.register!(TestRegistry, 'TS')
+        TestRegistry.register!(Integer, 'I')
+        map = TestRegistry.instance_variable_get("@serialization_map")
+        _(map).must_equal({'TS' => TestRegistry, 'I' => Integer})
+      end
+
+      it "prevents overwriting values" do
+        TestRegistry.register!(Integer, 'I')
+        _(-> { TestRegistry.register!(Float, 'I') }).must_raise Flows::Registry::IdentifierTaken
+      end
+
+      it "encodes and decodes values" do
+        TestRegistry.register!(Integer, 'I')
+        _(TestRegistry.encode(Integer)).must_equal 'I'
+      end
+
+      it "raises an exception when unknown key is requested" do
+        _(-> { TestRegistry.encode(Float) }).must_raise Flows::Registry::UnknownIdentifier
+        _(-> { TestRegistry.decode('F') }).must_raise Flows::Registry::UnknownIdentifier
+      end
+    end
+  end
+end

data/test/future_execution_test.rb

@@ -55,7 +55,7 @@ module Dynflow
 
         it 'delays the action' do
           _(execution_plan.steps.count).must_equal 1
-          _(delayed_plan.start_at.inspect).must_equal (@start_at).inspect
+          _(delayed_plan.start_at).must_be_within_delta(@start_at, 0.5)
           _(history_names.call(execution_plan)).must_equal ['delay']
         end
 

data/test/persistence_test.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require_relative 'test_helper'
 require 'tmpdir'
+require 'ostruct'
 
 module Dynflow
   module PersistenceTest
@@ -85,7 +86,7 @@ module Dynflow
         original.each do |key, value|
           loaded_value = loaded[key.to_s]
           if value.is_a?(Time)
-            _(loaded_value.inspect).must_equal value.inspect
+            _(loaded_value).must_be_within_delta(value, 0.5)
           elsif value.is_a?(Hash)
             assert_equal_attributes!(value, loaded_value)
           elsif value.nil?
@@ -347,7 +348,7 @@ module Dynflow
               if value.nil?
                 assert_nil stored.fetch(name.to_sym)
               elsif value.is_a?(Time)
-                _(stored.fetch(name.to_sym).inspect).must_equal value.inspect
+                _(stored.fetch(name.to_sym)).must_be_within_delta(value, 0.5)
               else
                 _(stored.fetch(name.to_sym)).must_equal value
               end
@@ -371,6 +372,41 @@ module Dynflow
           assert_equal [], adapter.pull_envelopes(executor_world_id)
         end
 
+        it 'supports pruning of envelopes of invalidated worlds' do
+          client_world_id   = '5678'
+          executor_world_id = '1234'
+          envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
+          executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
+          client_envelope   = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
+          envelopes         = [client_envelope, executor_envelope]
+
+          envelopes.each { |e| adapter.push_envelope(e) }
+
+          assert_equal 1, adapter.prune_envelopes([executor_world_id])
+          assert_equal 0, adapter.prune_envelopes([executor_world_id])
+          assert_equal [], adapter.pull_envelopes(executor_world_id)
+          assert_equal [client_envelope], adapter.pull_envelopes(client_world_id)
+        end
+
+        it 'supports pruning of orphaned envelopes' do
+          client_world_id   = '5678'
+          executor_world_id = '1234'
+          envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
+          executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
+          client_envelope   = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
+          envelopes         = [client_envelope, executor_envelope]
+
+          envelopes.each { |e| adapter.push_envelope(e) }
+          adapter.insert_coordinator_record({"class"=>"Dynflow::Coordinator::ExecutorWorld",
+                                             "id" => executor_world_id, "meta" => {}, "active" => true })
+
+          assert_equal 1, adapter.prune_undeliverable_envelopes
+          assert_equal 0, adapter.prune_undeliverable_envelopes
+          assert_equal [], adapter.pull_envelopes(client_world_id)
+          assert_equal [executor_envelope], adapter.pull_envelopes(executor_world_id)
+          assert_equal [], adapter.pull_envelopes(executor_world_id)
+        end
+
         it 'supports reading data saved prior to normalization' do
           db = adapter.send(:db)
           # Prepare records for saving

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.5.0
 platform: ruby
 authors:
 - Ivan Necas
 - Petr Chalupa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2021-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -242,10 +242,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/install_dependencies.sh"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
-- ".travis.yml"
 - Dockerfile
 - Gemfile
 - MIT-LICENSE
@@ -474,6 +475,7 @@ files:
 - lib/dynflow/flows/abstract_composed.rb
 - lib/dynflow/flows/atom.rb
 - lib/dynflow/flows/concurrence.rb
+- lib/dynflow/flows/registry.rb
 - lib/dynflow/flows/sequence.rb
 - lib/dynflow/logger_adapters.rb
 - lib/dynflow/logger_adapters/abstract.rb
@@ -512,6 +514,7 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb
@@ -575,11 +578,11 @@ files:
 - test/execution_plan_hooks_test.rb
 - test/execution_plan_test.rb
 - test/executor_test.rb
+- test/flows_test.rb
 - test/future_execution_test.rb
 - test/memory_cosumption_watcher_test.rb
 - test/middleware_test.rb
 - test/persistence_test.rb
-- test/prepare_travis_env.sh
 - test/redis_locking_test.rb
 - test/rescue_test.rb
 - test/round_robin_test.rb
@@ -624,7 +627,7 @@ homepage: https://github.com/Dynflow/dynflow
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -639,8 +642,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: DYNamic workFLOW engine
 test_files:
@@ -658,11 +661,11 @@ test_files:
 - test/execution_plan_hooks_test.rb
 - test/execution_plan_test.rb
 - test/executor_test.rb
+- test/flows_test.rb
 - test/future_execution_test.rb
 - test/memory_cosumption_watcher_test.rb
 - test/middleware_test.rb
 - test/persistence_test.rb
-- test/prepare_travis_env.sh
 - test/redis_locking_test.rb
 - test/rescue_test.rb
 - test/round_robin_test.rb

data/.travis.yml

@@ -1,33 +0,0 @@
-language: ruby
-
-services:
-  - postgresql
-  - redis
-
-rvm:
-  - "2.3.1"
-  - "2.4.0"
-  - "2.5.0"
-
-env:
-  global:
-    - "TESTOPTS=--verbose DB=postgresql DB_CONN_STRING=postgres://postgres@localhost/travis_ci_test"
-
-matrix:
-  include:
-    - rvm: "2.4.0"
-      env: "DB=mysql DB_CONN_STRING=mysql2://root@localhost/travis_ci_test"
-      services:
-        - mysql
-        - redis
-    - rvm: "2.4.0"
-      env: "DB=sqlite3 DB_CONN_STRING=sqlite:/"
-    - rvm: "2.4.0"
-      env: "CONCURRENT_RUBY_EXT=true"
-
-install:
-  - test/prepare_travis_env.sh
-
-script:
-  - bundle exec rubocop
-  - bundle exec rake test