dynflow 0.7.9 → 0.8.0

data/dynflow.gemspec

@@ -21,7 +21,9 @@ Gem::Specification.new do |s|
   s.add_dependency "activesupport"
   s.add_dependency "multi_json"
   s.add_dependency "apipie-params"
-  s.add_dependency "algebrick", '~> 0.4.0'
+  s.add_dependency "algebrick", '~> 0.7.0'
+  s.add_dependency "concurrent-ruby", '~> 0.9.0.pre3'
+  s.add_dependency "concurrent-ruby-edge"
 
   s.add_development_dependency "rack-test"
   s.add_development_dependency "minitest"

data/examples/example_helper.rb

@@ -8,9 +8,15 @@ class ExampleHelper
       @world ||= create_world
     end
 
-    def create_world(options = {})
-      options = default_world_options.merge(options)
-      Dynflow::SimpleWorld.new(options)
+    def create_world
+      config = Dynflow::Config.new
+      config.persistence_adapter = persistence_adapter
+      config.logger_adapter      = logger_adapter
+      config.auto_rescue         = false
+      yield config if block_given?
+      Dynflow::World.new(config).tap do |world|
+        puts "World #{world.id} started..."
+      end
     end
 
     def persistence_conn_string
@@ -21,22 +27,17 @@ class ExampleHelper
       Dynflow::PersistenceAdapters::Sequel.new persistence_conn_string
     end
 
-    def default_world_options
-      { logger_adapter: logger_adapter,
-        persistence_adapter: persistence_adapter }
-    end
-
     def logger_adapter
       Dynflow::LoggerAdapters::Simple.new $stderr, 4
     end
 
 
     def run_web_console(world = ExampleHelper.world)
-      require 'dynflow/web_console'
-      dynflow_console = Dynflow::WebConsole.setup do
+      require 'dynflow/web'
+      dynflow_console = Dynflow::Web.setup do
         set :world, world
       end
-      dynflow_console.run!
+      Rack::Server.new(:app => dynflow_console, :Port => 4567).start
     end
 
     # for simulation of the execution failing for the first time
@@ -52,5 +53,11 @@ class ExampleHelper
     def nothing_should_fail!
       @should_fail = false
     end
+
+    def terminate
+      @world.terminate.wait if @world
+    end
   end
 end
+
+at_exit { ExampleHelper.terminate }

data/examples/orchestrate_evented.rb

@@ -153,7 +153,8 @@ module OrchestrateEvented
         puts <<-MSG.gsub(/^.*\|/, '')
 
             | Execution plan #{execution_plan_id} got stuck
-            | You can cancel the stucked step at http://localhost:4567/#{execution_plan_id}
+            | You can cancel the stucked step at
+            | http://localhost:4567/#{execution_plan_id}
 
         MSG
         # we suspend the action but don't plan the wakeup event,

data/examples/remote_executor.rb

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 require_relative 'example_helper'
+require_relative 'orchestrate_evented'
 require 'tmpdir'
 
 class SampleAction < Dynflow::Action
@@ -19,37 +20,64 @@ end
 class RemoteExecutorExample
   class << self
 
-    def run_server
-      world               = ExampleHelper.create_world(persistence_adapter: persistence_adapter)
-      listener            = Dynflow::Listeners::Socket.new world, socket
-
-      Thread.new { Dynflow::Daemon.new(listener, world).run }
-      ExampleHelper.run_web_console(world)
-    ensure
-      File.delete(db_path)
+    def run_observer
+      world               = ExampleHelper.create_world do |config|
+        config.persistence_adapter = persistence_adapter
+        config.connector           = connector
+        config.executor            = false
+      end
+      run(world)
     end
 
-    def run_client
-      executor = ->(world) { Dynflow::Executors::RemoteViaSocket.new(world, socket) }
-      world    = ExampleHelper.create_world(persistence_adapter: persistence_adapter,
-                                            executor:            executor)
+    def run_server
+      world               = ExampleHelper.create_world do |config|
+        config.persistence_adapter = persistence_adapter
+        config.connector           = connector
+      end
+      run(world)
+    end
 
-      loop do
-        world.trigger(SampleAction).finished.wait
-        sleep 0.5
+    def run(world)
+      begin
+        ExampleHelper.run_web_console(world)
+      rescue Errno::EADDRINUSE
+        require 'io/console'
+        puts "Running without a web console. Press q<enter> to quit."
+        until STDIN.gets.chomp == 'q'
+        end
       end
     end
 
-    def socket
-      File.join(Dir.tmpdir, 'dynflow_socket')
+    def db_path
+      File.expand_path("../remote_executor_db.sqlite", __FILE__)
+    end
+
+    def persistence_conn_string
+      ENV['DB_CONN_STRING'] || "sqlite://#{db_path}"
     end
 
     def persistence_adapter
-      Dynflow::PersistenceAdapters::Sequel.new "sqlite://#{db_path}"
+      Dynflow::PersistenceAdapters::Sequel.new persistence_conn_string
     end
 
-    def db_path
-      File.expand_path("../remote_executor_db.sqlite", __FILE__)
+    def connector
+      Proc.new { |world| Dynflow::Connectors::Database.new(world) }
+    end
+
+    def run_client
+      world    = ExampleHelper.create_world do |config|
+        config.persistence_adapter = persistence_adapter
+        config.executor            = false
+        config.connector           = connector
+      end
+
+      world.trigger(OrchestrateEvented::CreateInfrastructure)
+      world.trigger(OrchestrateEvented::CreateInfrastructure, true)
+
+      loop do
+        world.trigger(SampleAction).finished.wait
+        sleep 0.5
+      end
     end
 
   end
@@ -59,6 +87,11 @@ command = ARGV.first || 'server'
 
 if $0 == __FILE__
   case command
+  when 'observer'
+    puts <<MSG
+The observer starting…. You can see what's going on there
+MSG
+   RemoteExecutorExample.run_observer
   when 'server'
     puts <<MSG
 The server is starting…. You can send the work to it by running:

data/lib/dynflow.rb

@@ -4,6 +4,14 @@ require 'thread'
 require 'set'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'base64'
+require 'concurrent'
+require 'concurrent-edge'
+
+logger                          = Logger.new($stderr)
+logger.level                    = Logger::INFO
+Concurrent.configuration.logger = lambda do |level, progname, message = nil, &block|
+  logger.add level, message, progname, &block
+end
 
 # TODO validate in/output, also validate unknown keys
 # TODO performance testing, how many actions will it handle?
@@ -14,23 +22,25 @@ module Dynflow
   class Error < StandardError
   end
 
+  require 'dynflow/round_robin'
+  require 'dynflow/actor'
   require 'dynflow/errors'
-  require 'dynflow/future'
-  require 'dynflow/micro_actor'
+  require 'dynflow/serializer'
   require 'dynflow/serializable'
   require 'dynflow/clock'
   require 'dynflow/stateful'
   require 'dynflow/transaction_adapters'
+  require 'dynflow/coordinator'
   require 'dynflow/persistence'
   require 'dynflow/middleware'
   require 'dynflow/flows'
+  require 'dynflow/execution_history'
   require 'dynflow/execution_plan'
   require 'dynflow/action'
-  require 'dynflow/listeners'
   require 'dynflow/executors'
   require 'dynflow/logger_adapters'
   require 'dynflow/world'
-  require 'dynflow/simple_world'
-  require 'dynflow/daemon'
-
+  require 'dynflow/connectors'
+  require 'dynflow/dispatcher'
+  require 'dynflow/config'
 end

data/lib/dynflow/action/suspended.rb

@@ -8,7 +8,7 @@ module Dynflow
       @step_id           = action.run_step_id
     end
 
-    def event(event, future = Future.new)
+    def event(event, future = Concurrent.future)
       @world.event execution_plan_id, step_id, event, future
     end
 

data/lib/dynflow/action/with_sub_plans.rb

@@ -51,9 +51,7 @@ module Dynflow
 
     # Helper for creating sub plans
     def trigger(*args)
-      world.trigger do
-        world.plan_with_caller(self, *args)
-      end
+      world.trigger { world.plan_with_caller(self, *args) }
     end
 
     def wait_for_sub_plans(sub_plans)
@@ -102,9 +100,8 @@ module Dynflow
     def notify_on_finish(plans)
       suspend do |suspended_action|
         plans.each do |plan|
-          plan.finished.do_then do |value|
-            suspended_action << SubPlanFinished[plan.execution_plan_id,
-                                                value.result == :success]
+          plan.finished.on_completion! do |success, value|
+            suspended_action << SubPlanFinished[plan.id, success && (value.result == :success)]
           end
         end
       end

data/lib/dynflow/actor.rb

@@ -0,0 +1,56 @@
+module Dynflow
+
+  module MethodicActor
+    def on_message(message)
+      method, *args = message
+      self.send(method, *args)
+    end
+  end
+
+  # Common parent for all the Dynflow actors defining some defaults
+  # that we preffer here.
+  class Actor < Concurrent::Actor::Context
+
+    include MethodicActor
+
+    # Behaviour that watches for polite asking for termination
+    # and calls corresponding method on the context to do so
+    class PoliteTermination < Concurrent::Actor::Behaviour::Abstract
+      def on_envelope(envelope)
+        message, terminated_future = envelope
+        if :start_termination == message
+          context.start_termination(terminated_future)
+          envelope.future.success true if !envelope.future.nil?
+          Concurrent::Actor::Behaviour::MESSAGE_PROCESSED
+        else
+          pass envelope
+        end
+      end
+    end
+
+    include Algebrick::Matching
+
+    def start_termination(future)
+      @terminated = future
+    end
+
+    def finish_termination
+      @terminated.success(true)
+      reference.tell(:terminate!)
+    end
+
+    def terminating?
+      !!@terminated
+    end
+
+    def behaviour_definition
+      [*Concurrent::Actor::Behaviour.base(:just_log),
+       Concurrent::Actor::Behaviour::Buffer,
+       [Concurrent::Actor::Behaviour::SetResults, :just_log],
+       Concurrent::Actor::Behaviour::Awaits,
+       PoliteTermination,
+       Concurrent::Actor::Behaviour::ExecutesContext,
+       Concurrent::Actor::Behaviour::ErrorsOnUnknownMessage]
+    end
+  end
+end

data/lib/dynflow/clock.rb

@@ -1,7 +1,7 @@
 module Dynflow
   require 'set'
 
-  class Clock < MicroActor
+  class Clock < Actor
 
     include Algebrick::Types
 
@@ -40,23 +40,7 @@ module Dynflow
                Pill = type { fields Float }
     end
 
-    def ping(who, time, with_what = nil, where = :<<)
-      Type! time, Time, Numeric
-      time  = Time.now + time if time.is_a? Numeric
-      timer = Timer[who, time, with_what.nil? ? Algebrick::Types::None : Some[Object][with_what], where]
-      if terminated?
-        Thread.new do
-          sleep [timer.when - Time.now, 0].max
-          timer.apply
-        end
-      else
-        self << timer
-      end
-    end
-
-    private
-
-    def delayed_initialize
+    def initialize
       @timers        = SortedSet.new
       @sleeping_pill = None
       @sleep_barrier = Mutex.new
@@ -64,27 +48,32 @@ module Dynflow
       Thread.pass until @sleep_barrier.locked? || @sleeper.status == 'sleep'
     end
 
-    def termination
-      @sleeper.kill
-      super
+    def default_reference_class
+      ClockReference
     end
 
-    def on_message(message)
-      match message,
-            Tick >-> do
-              run_ready_timers
-              sleep_to first_timer
-            end,
-            ~Timer >-> timer do
-              @timers.add timer
-              if @timers.size == 1
-                sleep_to timer
-              else
-                wakeup if timer == first_timer
-              end
-            end
+    def on_event(event)
+      if event == :terminated
+        @sleeper.kill
+      end
     end
 
+    def tick
+      run_ready_timers
+      sleep_to first_timer
+    end
+
+    def add_timer(timer)
+      @timers.add timer
+      if @timers.size == 1
+        sleep_to timer
+      else
+        wakeup if timer == first_timer
+      end
+    end
+
+    private
+
     def run_ready_timers
       while first_timer && first_timer.when <= Time.now
         first_timer.apply
@@ -122,12 +111,28 @@ module Dynflow
           pill           = @sleeping_pill
           @sleeping_pill = Took
           @sleep_barrier.sleep pill.value
-          self << Tick
+          reference.tell(:tick)
         end
       end
     end
-
   end
-end
 
+  class ClockReference < Concurrent::Actor::Reference
+    include Algebrick::Types
+
+    def ping(who, time, with_what = nil, where = :<<)
+      Type! time, Time, Numeric
+      time  = Time.now + time if time.is_a? Numeric
+      timer = Clock::Timer[who, time, with_what.nil? ? Algebrick::Types::None : Some[Object][with_what], where]
+      # if self.ask!(:terminated?) # FIXME not thread safe
+      #   Thread.new do
+      #     sleep [timer.when - Time.now, 0].max
+      #     timer.apply
+      #   end
+      # else
+      self.tell([:add_timer, timer])
+      # end
+    end
+  end
 
+end

data/lib/dynflow/config.rb

@@ -0,0 +1,107 @@
+require 'socket'
+
+module Dynflow
+  class Config
+    include Algebrick::TypeCheck
+
+    def self.config_attr(name, *types, &default)
+      self.send(:define_method, "validate_#{ name }!") do |value|
+        Type! value, *types unless types.empty?
+      end
+      self.send(:define_method, name) do
+        var_name = "@#{ name }"
+        if instance_variable_defined?(var_name)
+          return instance_variable_get(var_name)
+        else
+          return default
+        end
+      end
+      self.send(:attr_writer, name)
+    end
+
+    class ForWorld
+      attr_reader :world, :config
+
+      def initialize(config, world)
+        @config = config
+        @world  = world
+        @cache  = {}
+      end
+
+      def validate
+        @config.validate(self)
+      end
+
+      def method_missing(name)
+        return @cache[name] if @cache.key?(name)
+        value = @config.send(name)
+        value = value.call(@world, self) if value.is_a? Proc
+        @config.send("validate_#{ name }!", value)
+        @cache[name] = value
+      end
+    end
+
+    config_attr :logger_adapter, LoggerAdapters::Abstract do
+      LoggerAdapters::Simple.new
+    end
+
+    config_attr :transaction_adapter, TransactionAdapters::Abstract do
+      TransactionAdapters::None.new
+    end
+
+    config_attr :persistence_adapter, PersistenceAdapters::Abstract do
+      PersistenceAdapters::Sequel.new('sqlite:/')
+    end
+
+    config_attr :coordinator_adapter, CoordinatorAdapters::Abstract do |world|
+      CoordinatorAdapters::Sequel.new(world)
+    end
+
+    config_attr :pool_size, Fixnum do
+      5
+    end
+
+    config_attr :executor, Executors::Abstract, FalseClass do |world, config|
+      Executors::Parallel.new(world, config.pool_size)
+    end
+
+    config_attr :connector, Connectors::Abstract do |world|
+      Connectors::Direct.new(world)
+    end
+
+    config_attr :auto_rescue, Algebrick::Types::Boolean do
+      false
+    end
+
+    config_attr :exit_on_terminate, Algebrick::Types::Boolean do
+      true
+    end
+
+    config_attr :auto_terminate, Algebrick::Types::Boolean do
+      true
+    end
+
+    config_attr :auto_execute, Algebrick::Types::Boolean do
+      true
+    end
+
+    config_attr :action_classes do
+      Action.all_children
+    end
+
+    config_attr :meta do
+      { 'hostname' => Socket.gethostname, 'pid' => Process.pid }
+    end
+
+    def validate(config_for_world)
+      if defined? ::ActiveRecord::Base
+        ar_pool_size = ::ActiveRecord::Base.connection_pool.instance_variable_get(:@size)
+        if (config_for_world.pool_size / 2.0) > ar_pool_size
+          config_for_world.world.logger.warn 'Consider increasing ActiveRecord::Base.connection_pool size, ' +
+              "it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
+                              'threads in Dynflow pool.'
+        end
+      end
+    end
+  end
+end