dynflow 1.4.8 → 1.6.2

data/lib/dynflow/persistence_adapters/sequel_migrations/023_sqlite_workarounds.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+tables = [:dynflow_actions, :dynflow_delayed_plans, :dynflow_steps, :dynflow_output_chunks]
+Sequel.migration do
+  up do
+    if database_type == :sqlite && Gem::Version.new(SQLite3::SQLITE_VERSION) <= Gem::Version.new('3.7.17')
+      tables.each do |table|
+        alter_table(table) { drop_foreign_key [:execution_plan_uuid] }
+      end
+    end
+  end
+
+  down do
+    if database_type == :sqlite && Gem::Version.new(SQLite3::SQLITE_VERSION) <= Gem::Version.new('3.7.17')
+      tables.each do |table|
+        alter_table(table) { add_foreign_key [:execution_plan_uuid], :dynflow_execution_plans }
+      end
+    end
+  end
+end

data/lib/dynflow/serializable.rb

@@ -45,12 +45,12 @@ module Dynflow
       if values.size == 1
         value = values.first
         case value
-        when String, Numeric, Symbol, TrueClass, FalseClass, NilClass, Time
-          value
         when Hash
           value.inject({}) { |h, (k, v)| h.update k => recursive_to_hash(v) }
         when Array
           value.map { |v| recursive_to_hash v }
+        when ->(v) { v.respond_to?(:to_msgpack) }
+          value
         else
           value.to_hash
         end

data/lib/dynflow/testing/dummy_coordinator.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+module Dynflow
+  module Testing
+    class DummyCoordinator
+      def find_records(*args)
+        []
+      end
+    end
+  end
+end

data/lib/dynflow/testing/dummy_planned_action.rb

@@ -15,6 +15,10 @@ module Dynflow
         @plan_input = args
         self
       end
+
+      def run_step_id
+        @run_step_id ||= Testing.get_id
+      end
     end
   end
 end

data/lib/dynflow/testing/dummy_world.rb

@@ -5,7 +5,7 @@ module Dynflow
       extend Mimic
       mimic! World
 
-      attr_reader :clock, :executor, :middleware
+      attr_reader :clock, :executor, :middleware, :coordinator
       attr_accessor :action
 
       def initialize(_config = nil)
@@ -13,6 +13,7 @@ module Dynflow
         @clock          = ManagedClock.new
         @executor       = DummyExecutor.new(self)
         @middleware     = Middleware::World.new
+        @coordinator    = DummyCoordinator.new
       end
 
       def action_logger

data/lib/dynflow/testing/in_thread_executor.rb

@@ -34,8 +34,8 @@ module Dynflow
         @director.work_finished(work_item)
       end
 
-      def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
-        event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future])
+      def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future, optional: false)
+        event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future, optional])
         @director.handle_event(event).each do |work_item|
           @work_items << work_item
         end

data/lib/dynflow/testing/in_thread_world.rb

@@ -58,15 +58,15 @@ module Dynflow
         future.reject e
       end
 
-      def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
-        @executor.event(execution_plan_id, step_id, event, done)
+      def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
+        @executor.event(execution_plan_id, step_id, event, done, optional: optional)
       end
 
-      def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future)
+      def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future, optional: false)
         if time.nil? || time < Time.now
-          event(execution_plan_id, step_id, event, done)
+          event(execution_plan_id, step_id, event, done, optional: optional)
         else
-          clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done], :delayed_event)
+          clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done, optional], :delayed_event)
         end
       end
     end

data/lib/dynflow/testing.rb

@@ -19,6 +19,7 @@ module Dynflow
 
     require 'dynflow/testing/mimic'
     require 'dynflow/testing/managed_clock'
+    require 'dynflow/testing/dummy_coordinator'
     require 'dynflow/testing/dummy_world'
     require 'dynflow/testing/dummy_executor'
     require 'dynflow/testing/dummy_execution_plan'

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.4.8'
+  VERSION = '1.6.2'
 end

data/lib/dynflow/world.rb

@@ -200,6 +200,14 @@ module Dynflow
       Scheduled[execution_plan.id]
     end
 
+    def plan_elsewhere(action_class, *args)
+      execution_plan = ExecutionPlan.new(self, nil)
+      execution_plan.delay(nil, action_class, {}, *args)
+      plan_request(execution_plan.id)
+
+      Scheduled[execution_plan.id]
+    end
+
     def plan(action_class, *args)
       plan_with_options(action_class: action_class, args: args)
     end
@@ -219,12 +227,16 @@ module Dynflow
       publish_request(Dispatcher::Execution[execution_plan_id], done, true)
     end
 
-    def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
-      publish_request(Dispatcher::Event[execution_plan_id, step_id, event], done, false)
+    def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
+      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, nil, optional], done, false)
+    end
+
+    def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future, optional: false)
+      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time, optional], accepted, false)
     end
 
-    def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future)
-      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time], accepted, false)
+    def plan_request(execution_plan_id, done = Concurrent::Promises.resolvable_future)
+      publish_request(Dispatcher::Planning[execution_plan_id], done, false)
     end
 
     def ping(world_id, timeout, done = Concurrent::Promises.resolvable_future)

data/lib/dynflow.rb

@@ -72,11 +72,12 @@ module Dynflow
   require 'dynflow/throttle_limiter'
   require 'dynflow/telemetry'
   require 'dynflow/config'
+  require 'dynflow/extensions'
 
   if defined? ::ActiveJob
     require 'dynflow/active_job/queue_adapter'
 
-    class Railtie < Rails::Railtie
+    class Railtie < ::Rails::Railtie
       config.before_initialize do
         ::ActiveJob::QueueAdapters.send(
           :include,

data/test/dispatcher_test.rb

@@ -49,6 +49,12 @@ module Dynflow
               plan = result.finished.value
               assert_equal('finish', plan.actions.first.output[:event])
             end
+
+            it 'does not error on dispatching an optional event' do
+              request = client_world.event('123', 1, nil, optional: true)
+              request.wait(20)
+              assert_match /Could not find an executor for optional .*, discarding/, request.reason.message
+            end
           end
         end
       end

data/test/execution_plan_hooks_test.rb

@@ -60,6 +60,18 @@ module Dynflow
         execution_plan_hooks.use :raise_flag_root_only, :on => :stopped
       end
 
+      class PendingAction < ::Dynflow::Action
+        include FlagHook
+
+        execution_plan_hooks.use :raise_flag, :on => :pending
+      end
+
+      class AllTransitionsAction < ::Dynflow::Action
+        include FlagHook
+
+        execution_plan_hooks.use :raise_flag
+      end
+
       class ComposedAction < RootOnlyAction
         def plan
           plan_action(RootOnlyAction)
@@ -161,6 +173,30 @@ module Dynflow
         plan.finished.wait!
         _(Flag.raised_count).must_equal 1
       end
+
+      it 'runs the pending hooks when execution plan is created' do
+        refute Flag.raised?
+        plan = world.trigger(PendingAction)
+        plan.finished.wait!
+        _(Flag.raised_count).must_equal 1
+      end
+
+      it 'runs the pending hooks when execution plan is created' do
+        refute Flag.raised?
+        delay = world.delay(PendingAction, { :start_at => Time.now.utc + 180 })
+        delayed_plan = world.persistence.load_delayed_plan(delay.execution_plan_id)
+        delayed_plan.execution_plan.cancel.each(&:wait)
+        _(Flag.raised_count).must_equal 1
+      end
+
+      it 'runs the hook on every state transition' do
+        refute Flag.raised?
+        plan = world.trigger(AllTransitionsAction)
+        plan.finished.wait!
+        # There should be 5 transitions
+        # nothing -> pending -> planning -> planned -> running -> stopped
+        _(Flag.raised_count).must_equal 5
+      end
     end
   end
 end

data/test/extensions_test.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require_relative 'test_helper'
+require 'active_support/time'
+
+module Dynflow
+  module ExtensionsTest
+    describe 'msgpack extensions' do
+      before do
+        Thread.current[:time_zone] = ActiveSupport::TimeZone['Europe/Prague']
+      end
+      after { Thread.current[:time_zone] = nil }
+
+      it 'allows {de,}serializing Time' do
+        time = Time.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing ActiveSupport::TimeWithZone' do
+        time = Time.zone.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing DateTime' do
+        time = DateTime.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing Date' do
+        date = DateTime.current
+        transformed = MessagePack.unpack(date.to_msgpack)
+        assert_equal transformed, date
+        assert_equal transformed.class, date.class
+      end
+    end
+  end
+end

data/test/flows_test.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+require_relative 'test_helper'
+require 'mocha/minitest'
+
+module Dynflow
+  describe 'flow' do
+
+    class TestRegistry < Flows::Registry
+      class << self
+        def reset!
+          @serialization_map = {}
+        end
+      end
+    end
+
+    after do
+      TestRegistry.reset!
+    end
+
+    describe "registry" do
+      it "allows registering values" do
+        TestRegistry.register!(TestRegistry, 'TS')
+        TestRegistry.register!(Integer, 'I')
+        map = TestRegistry.instance_variable_get("@serialization_map")
+        _(map).must_equal({'TS' => TestRegistry, 'I' => Integer})
+      end
+
+      it "prevents overwriting values" do
+        TestRegistry.register!(Integer, 'I')
+        _(-> { TestRegistry.register!(Float, 'I') }).must_raise Flows::Registry::IdentifierTaken
+      end
+
+      it "encodes and decodes values" do
+        TestRegistry.register!(Integer, 'I')
+        _(TestRegistry.encode(Integer)).must_equal 'I'
+      end
+
+      it "raises an exception when unknown key is requested" do
+        _(-> { TestRegistry.encode(Float) }).must_raise Flows::Registry::UnknownIdentifier
+        _(-> { TestRegistry.decode('F') }).must_raise Flows::Registry::UnknownIdentifier
+      end
+    end
+  end
+end

data/test/future_execution_test.rb

@@ -29,14 +29,17 @@ module Dynflow
         describe 'abstract executor' do
           let(:abstract_delayed_executor) { DelayedExecutors::AbstractCore.new(world) }
 
-          it 'handles wrong plan state' do
+          it 'handles plan in planning state' do
             delayed_plan.execution_plan.state = :planning
             abstract_delayed_executor.send(:process, [delayed_plan], @start_at)
-            _(delayed_plan.execution_plan.state).must_equal :planned
+            _(delayed_plan.execution_plan.state).must_equal :scheduled
+          end
 
+          it 'handles plan in running state' do
             delayed_plan.execution_plan.set_state(:running, true)
             abstract_delayed_executor.send(:process, [delayed_plan], @start_at)
             _(delayed_plan.execution_plan.state).must_equal :running
+            _(world.persistence.load_delayed_plan(delayed_plan.execution_plan_uuid)).must_be :nil?
           end
         end
 
@@ -55,7 +58,7 @@ module Dynflow
 
         it 'delays the action' do
           _(execution_plan.steps.count).must_equal 1
-          _(delayed_plan.start_at.inspect).must_equal (@start_at).inspect
+          _(delayed_plan.start_at.to_i).must_equal(@start_at.to_i)
           _(history_names.call(execution_plan)).must_equal ['delay']
         end
 

data/test/persistence_test.rb

@@ -86,7 +86,7 @@ module Dynflow
         original.each do |key, value|
           loaded_value = loaded[key.to_s]
           if value.is_a?(Time)
-            _(loaded_value.inspect).must_equal value.inspect
+            _(loaded_value).must_be_within_delta(value, 0.5)
           elsif value.is_a?(Hash)
             assert_equal_attributes!(value, loaded_value)
           elsif value.nil?
@@ -348,7 +348,7 @@ module Dynflow
               if value.nil?
                 assert_nil stored.fetch(name.to_sym)
               elsif value.is_a?(Time)
-                _(stored.fetch(name.to_sym).inspect).must_equal value.inspect
+                _(stored.fetch(name.to_sym)).must_be_within_delta(value, 0.5)
               else
                 _(stored.fetch(name.to_sym)).must_equal value
               end

data/web/views/flow_step.erb

@@ -31,6 +31,7 @@
   <% end %>
   <%= show_action_data("Input:", action.input) %>
   <%= show_action_data("Output:", action.output) %>
+  <%= show_action_data("Chunked output:", action.stored_output_chunks) %>
   <% if step.error %>
     <p>
       <b>Error:</b>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.4.8
+  version: 1.6.2
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-13 00:00:00.000000000 Z
+date: 2021-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -25,6 +25,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.3
 - !ruby/object:Gem::Dependency
   name: apipie-params
   requirement: !ruby/object:Gem::Requirement
@@ -242,10 +262,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/install_dependencies.sh"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
-- ".travis.yml"
 - Dockerfile
 - Gemfile
 - MIT-LICENSE
@@ -386,6 +407,7 @@ files:
 - doc/pages/source/projects/index.md
 - docker-compose.yml
 - dynflow.gemspec
+- examples/chunked_output_benchmark.rb
 - examples/clock_benchmark.rb
 - examples/example_helper.rb
 - examples/future_execution.rb
@@ -397,6 +419,7 @@ files:
 - examples/sub_plan_concurrency_control.rb
 - examples/sub_plans.rb
 - examples/termination.rb
+- extras/expand/main.go
 - extras/statsd_mapping.conf
 - lib/dynflow.rb
 - lib/dynflow/action.rb
@@ -469,11 +492,14 @@ files:
 - lib/dynflow/executors/sidekiq/redis_locking.rb
 - lib/dynflow/executors/sidekiq/serialization.rb
 - lib/dynflow/executors/sidekiq/worker_jobs.rb
+- lib/dynflow/extensions.rb
+- lib/dynflow/extensions/msgpack.rb
 - lib/dynflow/flows.rb
 - lib/dynflow/flows/abstract.rb
 - lib/dynflow/flows/abstract_composed.rb
 - lib/dynflow/flows/atom.rb
 - lib/dynflow/flows/concurrence.rb
+- lib/dynflow/flows/registry.rb
 - lib/dynflow/flows/sequence.rb
 - lib/dynflow/logger_adapters.rb
 - lib/dynflow/logger_adapters/abstract.rb
@@ -513,6 +539,9 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/021_create_output_chunks.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/022_store_flows_as_msgpack.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/023_sqlite_workarounds.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb
@@ -534,6 +563,7 @@ files:
 - lib/dynflow/telemetry_adapters/statsd.rb
 - lib/dynflow/testing.rb
 - lib/dynflow/testing/assertions.rb
+- lib/dynflow/testing/dummy_coordinator.rb
 - lib/dynflow/testing/dummy_execution_plan.rb
 - lib/dynflow/testing/dummy_executor.rb
 - lib/dynflow/testing/dummy_planned_action.rb
@@ -576,11 +606,12 @@ files:
 - test/execution_plan_hooks_test.rb
 - test/execution_plan_test.rb
 - test/executor_test.rb
+- test/extensions_test.rb
+- test/flows_test.rb
 - test/future_execution_test.rb
 - test/memory_cosumption_watcher_test.rb
 - test/middleware_test.rb
 - test/persistence_test.rb
-- test/prepare_travis_env.sh
 - test/redis_locking_test.rb
 - test/rescue_test.rb
 - test/round_robin_test.rb
@@ -659,11 +690,12 @@ test_files:
 - test/execution_plan_hooks_test.rb
 - test/execution_plan_test.rb
 - test/executor_test.rb
+- test/extensions_test.rb
+- test/flows_test.rb
 - test/future_execution_test.rb
 - test/memory_cosumption_watcher_test.rb
 - test/middleware_test.rb
 - test/persistence_test.rb
-- test/prepare_travis_env.sh
 - test/redis_locking_test.rb
 - test/rescue_test.rb
 - test/round_robin_test.rb

data/.travis.yml

@@ -1,33 +0,0 @@
-language: ruby
-
-services:
-  - postgresql
-  - redis
-
-rvm:
-  - "2.3.1"
-  - "2.4.0"
-  - "2.5.0"
-
-env:
-  global:
-    - "TESTOPTS=--verbose DB=postgresql DB_CONN_STRING=postgres://postgres@localhost/travis_ci_test"
-
-matrix:
-  include:
-    - rvm: "2.4.0"
-      env: "DB=mysql DB_CONN_STRING=mysql2://root@localhost/travis_ci_test"
-      services:
-        - mysql
-        - redis
-    - rvm: "2.4.0"
-      env: "DB=sqlite3 DB_CONN_STRING=sqlite:/"
-    - rvm: "2.4.0"
-      env: "CONCURRENT_RUBY_EXT=true"
-
-install:
-  - test/prepare_travis_env.sh
-
-script:
-  - bundle exec rubocop
-  - bundle exec rake test