dynflow 0.8.16 → 0.8.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6082a4e18537f19b13eaf0409be81baaf334a5c
-  data.tar.gz: 5c9d56842579de336c97b7f00d4284ef254ba196
+  metadata.gz: 15cfcb2e4498b8f1032b1f36e3023aa469cbe313
+  data.tar.gz: 62d51d3df0f51ce41ab27692a1af5ca97ca88687
 SHA512:
-  metadata.gz: 6d24e6111ffac1b44316a31784afbe1ace05fd130e9c087bff718adfb33335fe2a7c747f176557b15999fd0b7900af1367128606452a7ce94c345894f5707f02
-  data.tar.gz: d7d3655784db8d993018d5c5e00c3dbebaf4679ae2bacf971eb129ea3e306854b97a803fa6098694ad5d79102aea2f70b456e481ec1af4a90469da71efe0f0a4
+  metadata.gz: 7e484ef3b5286395ca76547a08fab9d552ee558b7c3c0825bf262dfc53b7d48169aca85b615a954354c230826853343fb60c44ef63d6aa72d4f787ed811a7d41
+  data.tar.gz: 345656ec25bee35465bd8551108072cafd2ddfd0dd8ee10006da92e74ae1564eecd13f0b08e19649b0aa51a3458fb58ef06a690cbf2a2bc95a0eb6050f824956

data/.rubocop.yml

@@ -1 +1,22 @@
 inherit_from: .rubocop_todo.yml
+
+# Cop supports --auto-correct.
+# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
+Style/ExtraSpacing:
+  Enabled: true
+
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+# SupportedStyles: with_first_parameter, with_fixed_indentation
+Style/AlignParameters:
+  Enabled: true
+
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
+# SupportedStyles: aligned, indented
+Style/MultilineOperationIndentation:
+  Enabled: true
+
+# Cop supports --auto-correct.
+Style/EmptyLines:
+  Enabled: true

data/.rubocop_todo.yml

@@ -147,13 +147,6 @@ Style/Alias:
 Style/AlignHash:
   Enabled: false
 
-# Offense count: 51
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-# SupportedStyles: with_first_parameter, with_fixed_indentation
-Style/AlignParameters:
-  Enabled: false
-
 # Offense count: 11
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles.
@@ -255,11 +248,6 @@ Style/EmptyElse:
 Style/EmptyLineBetweenDefs:
   Enabled: false
 
-# Offense count: 14
-# Cop supports --auto-correct.
-Style/EmptyLines:
-  Enabled: false
-
 # Offense count: 1
 # Cop supports --auto-correct.
 Style/EmptyLinesAroundAccessModifier:
@@ -291,12 +279,6 @@ Style/EmptyLinesAroundMethodBody:
 Style/EmptyLinesAroundModuleBody:
   Enabled: false
 
-# Offense count: 25
-# Cop supports --auto-correct.
-# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
-Style/ExtraSpacing:
-  Enabled: false
-
 # Offense count: 9
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
@@ -395,13 +377,6 @@ Style/MultilineIfThen:
 Style/MultilineMethodCallIndentation:
   Enabled: false
 
-# Offense count: 9
-# Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
-# SupportedStyles: aligned, indented
-Style/MultilineOperationIndentation:
-  Enabled: false
-
 # Offense count: 9
 # Cop supports --auto-correct.
 Style/MutableConstant:

data/doc/pages/plugins/div_tag.rb

@@ -1,7 +1,7 @@
 module Jekyll
   class DivTag < Liquid::Block
     def render(context)
-      content            = super
+      content = super
 
       <<-HTML.gsub(/^ +\|/, '')
         |<#{tag} class="#{@markup}">

data/doc/pages/plugins/tags.rb

@@ -65,7 +65,6 @@ module Jekyll
         [tag.to_s, range < (size = posts.size) ? range = size : size]
       }
 
-
       range = 1..range
 
       tags.sort!.map! { |tag, size| [tag, range.quantile(size, num)] }

data/examples/orchestrate.rb

@@ -24,7 +24,6 @@ require_relative 'example_helper'
 
 module Orchestrate
 
-
   class CreateInfrastructure < Dynflow::Action
 
     def plan

data/examples/remote_executor.rb

@@ -21,7 +21,7 @@ class RemoteExecutorExample
   class << self
 
     def run_observer
-      world               = ExampleHelper.create_world do |config|
+      world = ExampleHelper.create_world do |config|
         config.persistence_adapter = persistence_adapter
         config.connector           = connector
         config.executor            = false
@@ -30,7 +30,7 @@ class RemoteExecutorExample
     end
 
     def run_server
-      world               = ExampleHelper.create_world do |config|
+      world = ExampleHelper.create_world do |config|
         config.persistence_adapter = persistence_adapter
         config.connector           = connector
       end
@@ -65,7 +65,7 @@ class RemoteExecutorExample
     end
 
     def run_client
-      world    = ExampleHelper.create_world do |config|
+      world = ExampleHelper.create_world do |config|
         config.persistence_adapter = persistence_adapter
         config.executor            = false
         config.connector           = connector

data/examples/sub_plan_concurrency_control.rb

@@ -38,7 +38,6 @@ class CostyAction < Dynflow::Action
   end
 end
 
-
 class ConcurrencyControlExample < Dynflow::Action
   include Dynflow::Action::WithSubPlans
 

data/examples/sub_plans.rb

@@ -16,7 +16,6 @@ DESC
 require_relative 'example_helper'
 require_relative 'orchestrate_evented'
 
-
 class SubPlansExample < Dynflow::Action
   include Dynflow::Action::WithSubPlans
 

data/lib/dynflow.rb

@@ -38,6 +38,7 @@ module Dynflow
   require 'dynflow/execution_plan'
   require 'dynflow/delayed_plan'
   require 'dynflow/action'
+  require 'dynflow/director'
   require 'dynflow/executors'
   require 'dynflow/logger_adapters'
   require 'dynflow/world'

data/lib/dynflow/action.rb

@@ -48,9 +48,10 @@ module Dynflow
       nil
     end
 
-    ERROR         = Object.new
-    SUSPEND       = Object.new
-    Phase         = Algebrick.type do
+    ERROR   = Object.new
+    SUSPEND = Object.new
+    Skip    = Algebrick.atom
+    Phase   = Algebrick.type do
       Executable = type do
         variants Plan     = atom,
                  Run      = atom,
@@ -58,7 +59,6 @@ module Dynflow
       end
       variants Executable, Present = atom
     end
-    Skip    = Algebrick.atom
 
     module Executable
       def execute_method_name
@@ -119,7 +119,7 @@ module Dynflow
 
     def phase!(*phases)
       phase?(*phases) or
-          raise TypeError, "Wrong phase #{phase}, required #{phases}"
+        raise TypeError, "Wrong phase #{phase}, required #{phases}"
     end
 
     def input=(hash)
@@ -137,7 +137,7 @@ module Dynflow
     def output
       if phase? Plan
         @output_reference or
-            raise 'plan_self has to be invoked before being able to reference the output'
+          raise 'plan_self has to be invoked before being able to reference the output'
       else
         @output
       end

data/lib/dynflow/config.rb

@@ -120,8 +120,8 @@ module Dynflow
         ar_pool_size = ::ActiveRecord::Base.connection_pool.instance_variable_get(:@size)
         if (config_for_world.pool_size / 2.0) > ar_pool_size
           config_for_world.world.logger.warn 'Consider increasing ActiveRecord::Base.connection_pool size, ' +
-              "it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
-                              'threads in Dynflow pool.'
+                                             "it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
+                                             'threads in Dynflow pool.'
         end
       end
     end

data/lib/dynflow/connectors/database.rb

@@ -151,7 +151,7 @@ module Dynflow
                                  1
                                end
                              end
-        @core  = Core.spawn('connector-database-core', self, polling_interval)
+        @core = Core.spawn('connector-database-core', self, polling_interval)
         start_listening(world) if world
       end
 

data/lib/dynflow/connectors/direct.rb

@@ -47,7 +47,7 @@ module Dynflow
       end
 
       def initialize(world = nil)
-        @core  = Core.spawn('connector-direct-core', self)
+        @core = Core.spawn('connector-direct-core', self)
         start_listening(world) if world
       end
 

data/lib/dynflow/coordinator.rb

@@ -60,8 +60,8 @@ module Dynflow
       # @api override
       # check to be performed before we try to acquire the lock
       def validate!
-        Type! id,       String
-        Type! @data,     Hash
+        Type! id,    String
+        Type! @data, Hash
         raise "The record id %{s} too large" % id if id.size > 100
         raise "The record class name %{s} too large" % self.class.name if self.class.name.size > 100
       end
@@ -94,7 +94,7 @@ module Dynflow
     class ExecutorWorld < WorldRecord
       def initialize(world)
         super
-        self.active    = !world.terminating?
+        self.active = !world.terminating?
       end
 
       def active?
@@ -143,7 +143,7 @@ module Dynflow
       def initialize(world)
         super
         @world = world
-        @data.merge!(owner_id: "world:#{world.id}",  world_id: world.id)
+        @data.merge!(owner_id: "world:#{world.id}", world_id: world.id)
       end
 
       def self.lock_id(*args)

data/lib/dynflow/director.rb

@@ -0,0 +1,190 @@
+module Dynflow
+  # Director is responsible for telling what to do next when:
+  #   * new execution starts
+  #   * an event accurs
+  #   * some work is finished
+  #
+  # It's public methods (except terminate) return work items that the
+  # executor should understand
+  class Director
+    include Algebrick::TypeCheck
+
+    Event = Algebrick.type do
+      fields! execution_plan_id: String,
+              step_id:           Fixnum,
+              event:             Object,
+              result:            Concurrent::Edge::Future
+    end
+
+    UnprocessableEvent = Class.new(Dynflow::Error)
+
+    class WorkItem
+      attr_reader :execution_plan_id
+
+      def initialize(execution_plan_id)
+        @execution_plan_id = execution_plan_id
+      end
+
+      def execute
+        raise NotImplementedError
+      end
+    end
+
+    class StepWorkItem < WorkItem
+      attr_reader :step
+
+      def initialize(execution_plan_id, step)
+        super(execution_plan_id)
+        @step = step
+      end
+
+      def execute
+        @step.execute(nil)
+      end
+    end
+
+    class EventWorkItem < StepWorkItem
+      attr_reader :event
+
+      def initialize(execution_plan_id, step, event)
+        super(execution_plan_id, step)
+        @event = event
+      end
+
+      def execute
+        @step.execute(@event.event)
+      end
+    end
+
+    class FinalizeWorkItem < WorkItem
+      def initialize(execution_plan_id, sequential_manager)
+        super(execution_plan_id)
+        @sequential_manager = sequential_manager
+      end
+
+      def execute
+        @sequential_manager.finalize
+      end
+    end
+
+    require 'dynflow/director/work_queue'
+    require 'dynflow/director/sequence_cursor'
+    require 'dynflow/director/flow_manager'
+    require 'dynflow/director/execution_plan_manager'
+    require 'dynflow/director/sequential_manager'
+    require 'dynflow/director/running_steps_manager'
+
+    attr_reader :logger
+
+    def initialize(world)
+      @world = world
+      @logger = world.logger
+      @execution_plan_managers = {}
+      @plan_ids_in_rescue = Set.new
+    end
+
+    def start_execution(execution_plan_id, finished)
+      manager = track_execution_plan(execution_plan_id, finished)
+      return [] unless manager
+      unless_done(manager, manager.start)
+    end
+
+    def handle_event(event)
+      Type! event, Event
+      execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
+      if execution_plan_manager
+        execution_plan_manager.event(event)
+      else
+        raise Dynflow::Error, "no manager for #{event.inspect}"
+      end
+    rescue Dynflow::Error => e
+      event.result.fail e.message
+      raise e
+    end
+
+    def work_finished(work)
+      manager = @execution_plan_managers[work.execution_plan_id]
+      unless_done(manager, manager.what_is_next(work))
+    end
+
+    def terminate
+      unless @execution_plan_managers.empty?
+        logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."
+        begin
+          @execution_plan_managers.values.each do |manager|
+            manager.terminate
+          end
+        rescue Errors::PersistenceError
+          logger.error "could not to clean the data properly"
+        end
+        @execution_plan_managers.values.each do |manager|
+          finish_manager(manager)
+        end
+      end
+    end
+
+    private
+
+    def unless_done(manager, work_items)
+      return [] unless manager
+      if manager.done?
+        finish_manager(manager)
+        return []
+      else
+        return work_items
+      end
+    end
+
+    def finish_manager(manager)
+      @execution_plan_managers.delete(manager.execution_plan.id)
+      if rescue?(manager)
+        rescue!(manager)
+      else
+        set_future(manager)
+      end
+    end
+
+    def rescue?(manager)
+      return false if @world.terminating?
+      @world.auto_rescue && manager.execution_plan.state == :paused &&
+        !@plan_ids_in_rescue.include?(manager.execution_plan.id)
+    end
+
+    def rescue!(manager)
+      # TODO: after moving to concurrent-ruby actors, there should be better place
+      # to put this logic of making sure we don't run rescues in endless loop
+      @plan_ids_in_rescue << manager.execution_plan.id
+      rescue_plan_id = manager.execution_plan.rescue_plan_id
+      if rescue_plan_id
+        @world.executor.execute(rescue_plan_id, manager.future, false)
+      else
+        set_future(manager)
+      end
+    end
+
+    def track_execution_plan(execution_plan_id, finished)
+      execution_plan = @world.persistence.load_execution_plan(execution_plan_id)
+
+      if @execution_plan_managers[execution_plan_id]
+        raise Dynflow::Error,
+              "cannot execute execution_plan_id:#{execution_plan_id} it's already running"
+      end
+
+      if execution_plan.state == :stopped
+        raise Dynflow::Error,
+              "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
+      end
+
+      @execution_plan_managers[execution_plan_id] =
+          ExecutionPlanManager.new(@world, execution_plan, finished)
+    rescue Dynflow::Error => e
+      finished.fail e
+      nil
+    end
+
+    def set_future(manager)
+      @plan_ids_in_rescue.delete(manager.execution_plan.id)
+      manager.future.success manager.execution_plan
+    end
+  end
+end