dynflow 1.4.7 → 1.6.1

data/lib/dynflow/persistence_adapters/sequel_migrations/021_create_output_chunks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+Sequel.migration do
+  up do
+    type = database_type
+    create_table(:dynflow_output_chunks) do
+      primary_key :id
+
+      column_properties = if type.to_s.include?('postgres')
+                            {type: :uuid}
+                          else
+                            {type: String, size: 36, fixed: true, null: false}
+                          end
+      foreign_key :execution_plan_uuid, :dynflow_execution_plans, **column_properties
+      index :execution_plan_uuid
+
+      column :action_id, Integer, null: false
+      foreign_key [:execution_plan_uuid, :action_id], :dynflow_actions,
+                  name: :dynflow_output_chunks_execution_plan_uuid_fkey1
+      index [:execution_plan_uuid, :action_id]
+
+      column :chunk, String, text: true
+      column :kind, String
+      column :timestamp, Time, null: false
+    end
+  end
+
+  down do
+    drop_table(:dynflow_output_chunks)
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/022_store_flows_as_msgpack.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'multi_json'
+require 'msgpack'
+
+def table_pkeys(table)
+  case table
+  when :dynflow_execution_plans
+    [:uuid]
+  when :dynflow_actions, :dynflow_steps
+    [:execution_plan_uuid, :id]
+  when :dynflow_coordinator_records
+    [:id, :class]
+  when :dynflow_delayed_plans
+    [:execution_plan_uuid]
+  when :dynflow_envelopes
+    [:id]
+  when :dynflow_output_chunks
+    [:chunk]
+  else
+    raise "Unknown table '#{table}'"
+  end
+end
+
+def conditions_for_row(table, row)
+  row.slice(*table_pkeys(table))
+end
+
+def migrate_table(table, from_names, to_names, new_type)
+  alter_table(table) do
+    to_names.each do |new|
+      add_column new, new_type
+    end
+  end
+
+  relevant_columns = table_pkeys(table) | from_names
+
+  from(table).select(*relevant_columns).each do |row|
+    update = from_names.zip(to_names).reduce({}) do |acc, (from, to)|
+      row[from].nil? ? acc : acc.merge(to => yield(row[from]))
+    end
+    next if update.empty?
+    from(table).where(conditions_for_row(table, row)).update(update)
+  end
+
+  from_names.zip(to_names).each do |old, new|
+    alter_table(table) do
+      drop_column old
+    end
+
+    if database_type == :mysql
+      type = new_type == File ? 'blob' : 'mediumtext'
+      run "ALTER TABLE #{table} CHANGE COLUMN `#{new}` `#{old}` #{type};"
+    else
+      rename_column table, new, old
+    end
+  end
+end
+
+Sequel.migration do
+
+  TABLES = {
+    :dynflow_actions => [:data, :input, :output],
+    :dynflow_coordinator_records => [:data],
+    :dynflow_delayed_plans => [:serialized_args, :data],
+    :dynflow_envelopes => [:data],
+    :dynflow_execution_plans => [:run_flow, :finalize_flow, :execution_history, :step_ids],
+    :dynflow_steps => [:error, :children],
+    :dynflow_output_chunks => [:chunk]
+  }
+
+  up do
+    TABLES.each do |table, columns|
+      new_columns = columns.map { |c| "#{c}_blob" }
+
+      migrate_table table, columns, new_columns, File do |data|
+        ::Sequel.blob(MessagePack.pack(MultiJson.load(data)))
+      end
+    end
+  end
+
+  down do
+    TABLES.each do |table, columns|
+      new_columns = columns.map { |c| c + '_text' }
+      migrate_table table, columns, new_columns, String do |data|
+        MultiJson.dump(MessagePack.unpack(data))
+      end
+    end
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/023_sqlite_workarounds.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+tables = [:dynflow_actions, :dynflow_delayed_plans, :dynflow_steps, :dynflow_output_chunks]
+Sequel.migration do
+  up do
+    if database_type == :sqlite && Gem::Version.new(SQLite3::SQLITE_VERSION) <= Gem::Version.new('3.7.17')
+      tables.each do |table|
+        alter_table(table) { drop_foreign_key [:execution_plan_uuid] }
+      end
+    end
+  end
+
+  down do
+    if database_type == :sqlite && Gem::Version.new(SQLite3::SQLITE_VERSION) <= Gem::Version.new('3.7.17')
+      tables.each do |table|
+        alter_table(table) { add_foreign_key [:execution_plan_uuid], :dynflow_execution_plans }
+      end
+    end
+  end
+end

data/lib/dynflow/serializable.rb

@@ -45,12 +45,12 @@ module Dynflow
       if values.size == 1
         value = values.first
         case value
-        when String, Numeric, Symbol, TrueClass, FalseClass, NilClass, Time
-          value
         when Hash
           value.inject({}) { |h, (k, v)| h.update k => recursive_to_hash(v) }
         when Array
           value.map { |v| recursive_to_hash v }
+        when ->(v) { v.respond_to?(:to_msgpack) }
+          value
         else
           value.to_hash
         end

data/lib/dynflow/testing/dummy_coordinator.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+module Dynflow
+  module Testing
+    class DummyCoordinator
+      def find_records(*args)
+        []
+      end
+    end
+  end
+end

data/lib/dynflow/testing/dummy_planned_action.rb

@@ -15,6 +15,10 @@ module Dynflow
         @plan_input = args
         self
       end
+
+      def run_step_id
+        @run_step_id ||= Testing.get_id
+      end
     end
   end
 end

data/lib/dynflow/testing/dummy_world.rb

@@ -5,7 +5,7 @@ module Dynflow
       extend Mimic
       mimic! World
 
-      attr_reader :clock, :executor, :middleware
+      attr_reader :clock, :executor, :middleware, :coordinator
       attr_accessor :action
 
       def initialize(_config = nil)
@@ -13,6 +13,7 @@ module Dynflow
         @clock          = ManagedClock.new
         @executor       = DummyExecutor.new(self)
         @middleware     = Middleware::World.new
+        @coordinator    = DummyCoordinator.new
       end
 
       def action_logger

data/lib/dynflow/testing/in_thread_executor.rb

@@ -34,8 +34,8 @@ module Dynflow
         @director.work_finished(work_item)
       end
 
-      def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
-        event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future])
+      def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future, optional: false)
+        event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future, optional])
         @director.handle_event(event).each do |work_item|
           @work_items << work_item
         end

data/lib/dynflow/testing/in_thread_world.rb

@@ -58,15 +58,15 @@ module Dynflow
         future.reject e
       end
 
-      def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
-        @executor.event(execution_plan_id, step_id, event, done)
+      def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
+        @executor.event(execution_plan_id, step_id, event, done, optional: optional)
       end
 
-      def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future)
+      def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future, optional: false)
         if time.nil? || time < Time.now
-          event(execution_plan_id, step_id, event, done)
+          event(execution_plan_id, step_id, event, done, optional: optional)
         else
-          clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done], :delayed_event)
+          clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done, optional], :delayed_event)
         end
       end
     end

data/lib/dynflow/testing.rb

@@ -19,6 +19,7 @@ module Dynflow
 
     require 'dynflow/testing/mimic'
     require 'dynflow/testing/managed_clock'
+    require 'dynflow/testing/dummy_coordinator'
     require 'dynflow/testing/dummy_world'
     require 'dynflow/testing/dummy_executor'
     require 'dynflow/testing/dummy_execution_plan'

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.4.7'
+  VERSION = '1.6.1'
 end

data/lib/dynflow/world.rb

@@ -200,6 +200,14 @@ module Dynflow
       Scheduled[execution_plan.id]
     end
 
+    def plan_elsewhere(action_class, *args)
+      execution_plan = ExecutionPlan.new(self, nil)
+      execution_plan.delay(nil, action_class, {}, *args)
+      plan_request(execution_plan.id)
+
+      Scheduled[execution_plan.id]
+    end
+
     def plan(action_class, *args)
       plan_with_options(action_class: action_class, args: args)
     end
@@ -219,12 +227,16 @@ module Dynflow
       publish_request(Dispatcher::Execution[execution_plan_id], done, true)
     end
 
-    def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
-      publish_request(Dispatcher::Event[execution_plan_id, step_id, event], done, false)
+    def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
+      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, nil, optional], done, false)
+    end
+
+    def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future, optional: false)
+      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time, optional], accepted, false)
     end
 
-    def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future)
-      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time], accepted, false)
+    def plan_request(execution_plan_id, done = Concurrent::Promises.resolvable_future)
+      publish_request(Dispatcher::Planning[execution_plan_id], done, false)
     end
 
     def ping(world_id, timeout, done = Concurrent::Promises.resolvable_future)

data/lib/dynflow.rb

@@ -72,11 +72,12 @@ module Dynflow
   require 'dynflow/throttle_limiter'
   require 'dynflow/telemetry'
   require 'dynflow/config'
+  require 'dynflow/extensions'
 
   if defined? ::ActiveJob
     require 'dynflow/active_job/queue_adapter'
 
-    class Railtie < Rails::Railtie
+    class Railtie < ::Rails::Railtie
       config.before_initialize do
         ::ActiveJob::QueueAdapters.send(
           :include,

data/test/dispatcher_test.rb

@@ -49,6 +49,12 @@ module Dynflow
               plan = result.finished.value
               assert_equal('finish', plan.actions.first.output[:event])
             end
+
+            it 'does not error on dispatching an optional event' do
+              request = client_world.event('123', 1, nil, optional: true)
+              request.wait(20)
+              assert_match /Could not find an executor for optional .*, discarding/, request.reason.message
+            end
           end
         end
       end

data/test/execution_plan_hooks_test.rb

@@ -60,6 +60,18 @@ module Dynflow
         execution_plan_hooks.use :raise_flag_root_only, :on => :stopped
       end
 
+      class PendingAction < ::Dynflow::Action
+        include FlagHook
+
+        execution_plan_hooks.use :raise_flag, :on => :pending
+      end
+
+      class AllTransitionsAction < ::Dynflow::Action
+        include FlagHook
+
+        execution_plan_hooks.use :raise_flag
+      end
+
       class ComposedAction < RootOnlyAction
         def plan
           plan_action(RootOnlyAction)
@@ -161,6 +173,30 @@ module Dynflow
         plan.finished.wait!
         _(Flag.raised_count).must_equal 1
       end
+
+      it 'runs the pending hooks when execution plan is created' do
+        refute Flag.raised?
+        plan = world.trigger(PendingAction)
+        plan.finished.wait!
+        _(Flag.raised_count).must_equal 1
+      end
+
+      it 'runs the pending hooks when execution plan is created' do
+        refute Flag.raised?
+        delay = world.delay(PendingAction, { :start_at => Time.now.utc + 180 })
+        delayed_plan = world.persistence.load_delayed_plan(delay.execution_plan_id)
+        delayed_plan.execution_plan.cancel.each(&:wait)
+        _(Flag.raised_count).must_equal 1
+      end
+
+      it 'runs the hook on every state transition' do
+        refute Flag.raised?
+        plan = world.trigger(AllTransitionsAction)
+        plan.finished.wait!
+        # There should be 5 transitions
+        # nothing -> pending -> planning -> planned -> running -> stopped
+        _(Flag.raised_count).must_equal 5
+      end
     end
   end
 end

data/test/extensions_test.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require_relative 'test_helper'
+require 'active_support/time'
+
+module Dynflow
+  module ExtensionsTest
+    describe 'msgpack extensions' do
+      before do
+        Thread.current[:time_zone] = ActiveSupport::TimeZone['Europe/Prague']
+      end
+      after { Thread.current[:time_zone] = nil }
+
+      it 'allows {de,}serializing Time' do
+        time = Time.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing ActiveSupport::TimeWithZone' do
+        time = Time.zone.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing DateTime' do
+        time = DateTime.now
+        transformed = MessagePack.unpack(time.to_msgpack)
+        assert_equal transformed, time
+        assert_equal transformed.class, time.class
+      end
+
+      it 'allows {de,}serializing Date' do
+        date = DateTime.current
+        transformed = MessagePack.unpack(date.to_msgpack)
+        assert_equal transformed, date
+        assert_equal transformed.class, date.class
+      end
+    end
+  end
+end

data/test/flows_test.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+require_relative 'test_helper'
+require 'mocha/minitest'
+
+module Dynflow
+  describe 'flow' do
+
+    class TestRegistry < Flows::Registry
+      class << self
+        def reset!
+          @serialization_map = {}
+        end
+      end
+    end
+
+    after do
+      TestRegistry.reset!
+    end
+
+    describe "registry" do
+      it "allows registering values" do
+        TestRegistry.register!(TestRegistry, 'TS')
+        TestRegistry.register!(Integer, 'I')
+        map = TestRegistry.instance_variable_get("@serialization_map")
+        _(map).must_equal({'TS' => TestRegistry, 'I' => Integer})
+      end
+
+      it "prevents overwriting values" do
+        TestRegistry.register!(Integer, 'I')
+        _(-> { TestRegistry.register!(Float, 'I') }).must_raise Flows::Registry::IdentifierTaken
+      end
+
+      it "encodes and decodes values" do
+        TestRegistry.register!(Integer, 'I')
+        _(TestRegistry.encode(Integer)).must_equal 'I'
+      end
+
+      it "raises an exception when unknown key is requested" do
+        _(-> { TestRegistry.encode(Float) }).must_raise Flows::Registry::UnknownIdentifier
+        _(-> { TestRegistry.decode('F') }).must_raise Flows::Registry::UnknownIdentifier
+      end
+    end
+  end
+end

data/test/future_execution_test.rb

@@ -29,14 +29,17 @@ module Dynflow
         describe 'abstract executor' do
           let(:abstract_delayed_executor) { DelayedExecutors::AbstractCore.new(world) }
 
-          it 'handles wrong plan state' do
+          it 'handles plan in planning state' do
             delayed_plan.execution_plan.state = :planning
             abstract_delayed_executor.send(:process, [delayed_plan], @start_at)
-            _(delayed_plan.execution_plan.state).must_equal :planned
+            _(delayed_plan.execution_plan.state).must_equal :scheduled
+          end
 
+          it 'handles plan in running state' do
             delayed_plan.execution_plan.set_state(:running, true)
             abstract_delayed_executor.send(:process, [delayed_plan], @start_at)
             _(delayed_plan.execution_plan.state).must_equal :running
+            _(world.persistence.load_delayed_plan(delayed_plan.execution_plan_uuid)).must_be :nil?
           end
         end
 
@@ -55,7 +58,7 @@ module Dynflow
 
         it 'delays the action' do
           _(execution_plan.steps.count).must_equal 1
-          _(delayed_plan.start_at.inspect).must_equal (@start_at).inspect
+          _(delayed_plan.start_at.to_i).must_equal(@start_at.to_i)
           _(history_names.call(execution_plan)).must_equal ['delay']
         end
 

data/test/persistence_test.rb

@@ -86,7 +86,7 @@ module Dynflow
         original.each do |key, value|
           loaded_value = loaded[key.to_s]
           if value.is_a?(Time)
-            _(loaded_value.inspect).must_equal value.inspect
+            _(loaded_value).must_be_within_delta(value, 0.5)
           elsif value.is_a?(Hash)
             assert_equal_attributes!(value, loaded_value)
           elsif value.nil?
@@ -348,7 +348,7 @@ module Dynflow
               if value.nil?
                 assert_nil stored.fetch(name.to_sym)
               elsif value.is_a?(Time)
-                _(stored.fetch(name.to_sym).inspect).must_equal value.inspect
+                _(stored.fetch(name.to_sym)).must_be_within_delta(value, 0.5)
               else
                 _(stored.fetch(name.to_sym)).must_equal value
               end

data/web/views/flow_step.erb

@@ -31,6 +31,7 @@
   <% end %>
   <%= show_action_data("Input:", action.input) %>
   <%= show_action_data("Output:", action.output) %>
+  <%= show_action_data("Chunked output:", action.stored_output_chunks) %>
   <% if step.error %>
     <p>
       <b>Error:</b>

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.4.7
+  version: 1.6.1
 platform: ruby
 authors:
 - Ivan Necas
 - Petr Chalupa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 1970-01-01 00:00:00.000000000 Z
+date: 2021-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -25,6 +25,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.3
 - !ruby/object:Gem::Dependency
   name: apipie-params
   requirement: !ruby/object:Gem::Requirement
@@ -242,10 +256,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/install_dependencies.sh"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
-- ".travis.yml"
 - Dockerfile
 - Gemfile
 - MIT-LICENSE
@@ -386,6 +401,7 @@ files:
 - doc/pages/source/projects/index.md
 - docker-compose.yml
 - dynflow.gemspec
+- examples/chunked_output_benchmark.rb
 - examples/clock_benchmark.rb
 - examples/example_helper.rb
 - examples/future_execution.rb
@@ -397,6 +413,7 @@ files:
 - examples/sub_plan_concurrency_control.rb
 - examples/sub_plans.rb
 - examples/termination.rb
+- extras/expand/main.go
 - extras/statsd_mapping.conf
 - lib/dynflow.rb
 - lib/dynflow/action.rb
@@ -469,11 +486,14 @@ files:
 - lib/dynflow/executors/sidekiq/redis_locking.rb
 - lib/dynflow/executors/sidekiq/serialization.rb
 - lib/dynflow/executors/sidekiq/worker_jobs.rb
+- lib/dynflow/extensions.rb
+- lib/dynflow/extensions/msgpack.rb
 - lib/dynflow/flows.rb
 - lib/dynflow/flows/abstract.rb
 - lib/dynflow/flows/abstract_composed.rb
 - lib/dynflow/flows/atom.rb
 - lib/dynflow/flows/concurrence.rb
+- lib/dynflow/flows/registry.rb
 - lib/dynflow/flows/sequence.rb
 - lib/dynflow/logger_adapters.rb
 - lib/dynflow/logger_adapters/abstract.rb
@@ -513,6 +533,9 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/021_create_output_chunks.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/022_store_flows_as_msgpack.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/023_sqlite_workarounds.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb
@@ -534,6 +557,7 @@ files:
 - lib/dynflow/telemetry_adapters/statsd.rb
 - lib/dynflow/testing.rb
 - lib/dynflow/testing/assertions.rb
+- lib/dynflow/testing/dummy_coordinator.rb
 - lib/dynflow/testing/dummy_execution_plan.rb
 - lib/dynflow/testing/dummy_executor.rb
 - lib/dynflow/testing/dummy_planned_action.rb
@@ -576,11 +600,12 @@ files:
 - test/execution_plan_hooks_test.rb
 - test/execution_plan_test.rb
 - test/executor_test.rb
+- test/extensions_test.rb
+- test/flows_test.rb
 - test/future_execution_test.rb
 - test/memory_cosumption_watcher_test.rb
 - test/middleware_test.rb
 - test/persistence_test.rb
-- test/prepare_travis_env.sh
 - test/redis_locking_test.rb
 - test/rescue_test.rb
 - test/round_robin_test.rb
@@ -625,7 +650,7 @@ homepage: https://github.com/Dynflow/dynflow
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -641,7 +666,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: DYNamic workFLOW engine
 test_files:
@@ -659,11 +684,12 @@ test_files:
 - test/execution_plan_hooks_test.rb
 - test/execution_plan_test.rb
 - test/executor_test.rb
+- test/extensions_test.rb
+- test/flows_test.rb
 - test/future_execution_test.rb
 - test/memory_cosumption_watcher_test.rb
 - test/middleware_test.rb
 - test/persistence_test.rb
-- test/prepare_travis_env.sh
 - test/redis_locking_test.rb
 - test/rescue_test.rb
 - test/round_robin_test.rb