dopi 0.17.0

data/lib/dopi/plan.rb

@@ -0,0 +1,99 @@
+#
+# This class loads a deployment plan
+#
+require 'forwardable'
+require 'yaml'
+require 'dop_common'
+require 'fileutils'
+
+module Dopi
+  class Plan
+    extend Forwardable
+    include Dopi::State
+
+    attr_reader :plan_parser, :version, :context_logger
+
+    def initialize(plan_parser)
+      @version = Dopi::VERSION
+      @plan_parser = plan_parser
+
+      step_sets.each{|step_set| state_add_child(step_set)}
+    end
+
+    def_delegators :@plan_parser,
+      :name,
+      :configuration,
+      :credentials,
+      :max_in_flight,
+      :max_per_role,
+      :canary_host
+
+    def run(options = {})
+      options_defaults = {
+        :run_for_nodes => :all,
+        :noop          => false,
+        :step_set      => 'default',
+        :node_info     => {},
+        :run_id        => Time.now.strftime('%Y%m%d-%H%M%S'),
+      }
+      run_options = options_defaults.merge(options)
+
+      context_log_path = File.join(DopCommon.config.log_dir, "#{run_options[:run_id]}-#{name}")
+      node_names = nodes.map{|n| n.name}
+      @context_logger = DopCommon::ThreadContextLogger.new(context_log_path, node_names)
+
+      nodes.each{|node| node.node_info = run_options[:node_info][node.fqdn] || {}}
+      step_set = step_sets.find{|s| s.name == run_options[:step_set]}
+      raise "Plan: Step set #{run_options[:step_set]} does not exist" if step_set.nil?
+      step_set.run(run_options)
+    ensure
+      @context_logger.cleanup
+    end
+
+    # The main validation work is done in the dop_common
+    # parser. We just add the command plugin parsers
+    def valid?
+      validity = @plan_parser.valid?
+      validity = false unless step_sets.all?{|step_set| step_set.valid? }
+      validity
+    rescue => e
+      DopCommon.config.trace ? Dopi.log.error(e) : Dopi.log.error(e.message)
+      Dopi.log.warn("Plan: Can't validate the command plugins because of a previous error")
+    end
+
+    def nodes
+      @nodes ||= parsed_nodes.map do |parsed_node|
+        ::Dopi::Node.new(parsed_node, self)
+      end
+    end
+
+    def step_sets
+      @step_sets ||= parsed_step_sets.map do |parsed_step_set|
+        ::Dopi::StepSet.new(parsed_step_set, self)
+      end
+    end
+
+    def load_state(state_hash)
+      if state_hash[:step_sets].kind_of?(Hash)
+        step_sets.each do |step_set|
+          step_set_state = state_hash[:step_sets][step_set.name] || []
+          step_set.load_state(step_set_state)
+        end
+      end
+    end
+
+    def state_hash
+      step_sets_hash = {}
+      step_sets.each do |step_set|
+        step_sets_hash[step_set.name] = step_set.state_hash
+      end
+      {:step_sets => step_sets_hash}
+    end
+
+  private
+
+    def_delegator  :@plan_parser, :nodes,     :parsed_nodes
+    def_delegator  :@plan_parser, :step_sets, :parsed_step_sets
+
+  end
+end

data/lib/dopi/pluginmanager.rb

@@ -0,0 +1,62 @@
+#
+# This class registers the plugins as they get loaded by ruby
+# and can create instances based on the plugin name
+#
+
+module Dopi
+  module PluginManager
+
+    @plugins = {}
+
+    def self.<<(plugin_klass)
+      plugin_name = get_plugin_name(plugin_klass)
+
+      raise Dopi::PluginLoaderError,
+        "Plugin class #{plugin_klass.to_s} (#{plugin_name}) already loaded" if @plugins[plugin_name]
+
+      @plugins[plugin_name] = plugin_klass
+    end
+
+    def self.create_instance(plugin_name, *args)
+      begin
+        @plugins[plugin_name].new(*args)
+      rescue Exception => e
+        raise PluginLoaderError, "Could not create instance of plugin #{plugin_name}: #{e.message}"
+      end
+    end
+
+    def self.get_plugin_name(plugin_klass)
+      plugin_klass.to_s.
+        gsub(/::/, '/').
+        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+        gsub(/([a-z\d])([A-Z])/,'\1_\2').
+        tr("-", "_").
+        downcase
+    end
+
+    # Expects a regular expression as a plugin filter
+    def self.plugin_list(filter = nil)
+      if filter
+        regexp = Regexp.new(filter)
+        @plugins.select{|p| p =~ regexp}
+      else
+        @plugins
+      end
+    end
+
+    # Expects a regular expression as a plugin filter
+    def self.plugin_name_list(filter = nil)
+      plugin_list(filter).keys
+    end
+
+    # Expects a regular expression as a plugin filter
+    def self.plugin_klass_list(filter = nil)
+      plugin_list(filter).values
+    end
+
+    def self.plugin_klass(plugin_name)
+      @plugins[plugin_name]
+    end
+
+  end
+end

data/lib/dopi/state.rb

@@ -0,0 +1,226 @@
+#
+# This is a simple hierarchical state tracker which chan keep
+# track of it's state based on it's children
+#
+require 'observer'
+
+module Dopi
+  module State
+    include Observable
+
+    def to_yaml_properties
+      instance_variables - [:@signal_procs]
+    end
+
+    def state
+      @state ||= :ready
+    end
+
+    def state_auto_evaluate_children
+      @auto_evaluate_children = true if @auto_evaluate_children.nil?
+      @auto_evaluate_children
+    end
+
+    def state_auto_evaluate_children=(value)
+      @auto_evaluate_children = value
+    end
+
+    def state_children
+      @state_children ||= []
+    end
+
+    def state_add_child(child)
+      state_children << child
+      child.add_observer(self)
+    end
+
+    def state_children_ready?
+      state_children.any? {|child| child.state_ready?}
+    end
+
+    def state_children_starting?
+      state_children.any? {|child| child.state_starting?}
+    end
+
+    def state_children_running_noop?
+      state_children.any? {|child| child.state_running_noop?}
+    end
+
+    def state_children_running?
+      state_children.any? {|child| child.state_running?}
+    end
+
+    def state_children_failed?
+      state_children.any? {|child| child.state_failed?}
+    end
+
+    def state_children_done?
+      state_children.all? {|child| child.state_done?}
+    end
+
+    def state_children_partial?
+      state_partial? || state_children.any?{|c| c.state_children_partial?}
+    end
+
+    def update_mutex
+      @upate_mutex || Mutex.new
+    end
+
+    def update(notify_only)
+      update_mutex.synchronize do
+        unless notify_only
+          old_state = @state
+          Dopi.log.debug("Checking if state of '#{name}' needs to be updated")
+          unless state_children.empty? || !state_auto_evaluate_children
+            if    state_children_failed?       then @state = :failed
+            elsif state_children_done?         then @state = :done
+            elsif state_children_running?      then @state = :running
+            elsif state_children_running_noop? then @state = :running_noop
+            elsif state_children_starting?     then @state = :starting
+            elsif state_children_ready?        then @state = :ready
+            end
+            if old_state == @state
+              state_changed(true)
+            else
+              state_changed(false)
+            end
+          end
+        else
+          state_changed(true)
+        end
+      end
+    end
+
+    def state_reset_with_children(force = false)
+      state_reset(force) if state_failed? or force
+      if state_children_failed? or force
+        state_children.each {|child| child.state_reset_with_children(force) }
+      end
+    end
+
+    def state_ready?
+      state == :ready
+    end
+
+    def state_starting?
+      state == :starting
+    end
+
+    def state_running?
+      state == :running
+    end
+
+    def state_running_noop?
+      state == :running_noop
+    end
+
+    def state_done?
+      state == :done
+    end
+
+    def state_failed?
+      state == :failed
+    end
+
+    def state_partial?
+      [:failed, :done, :running, :running_noop, :starting, :ready].each do |s|
+        return false if state_children.all?{|c| c.state == s}
+      end
+      return true
+    end
+
+    def state_start
+      return if state == :starting
+      raise Dopi::StateTransitionError, "Can't switch to running from #{state.to_s}" unless state == :ready
+      @state = :starting
+      state_changed
+    end
+
+    def state_run
+      return if state == :running
+      raise Dopi::StateTransitionError, "Can't switch to running from #{state.to_s}" unless state == :ready || state == :starting
+      @state = :running
+      state_changed
+    end
+
+    def state_run_noop
+      return if state == :running_noop
+      raise Dopi::StateTransitionError, "Can't switch to running_noop from #{state.to_s}" unless state == :ready || state == :starting
+      @state = :running_noop
+      state_changed
+    end
+
+    def state_ready
+      return if state == :ready
+      raise Dopi::StateTransitionError, "Can't switch to ready from #{state.to_s}" unless state == :running_noop
+      @state = :ready
+      state_changed
+    end
+
+    def state_finish
+      return if state == :done
+      raise Dopi::StateTransitionError, "Can't switch to done from #{state.to_s}" unless state == :running
+      @state = :done
+      state_changed
+    end
+
+    def state_fail
+      return if state == :failed
+      raise Dopi::StateTransitionError, "Can't switch to failed from #{state.to_s}" unless state == :running
+      @state = :failed
+      state_changed
+    end
+
+    def state_reset(force = false)
+      if force
+        state_children.each {|child| child.state_reset(force)}
+        @state = :ready
+        state_changed
+      else
+        raise Dopi::StateTransitionError, "Can't switch to ready from #{state.to_s}" unless state == :failed || state == :ready
+        if state_children.any?
+          state_children.each {|child| child.state_reset unless (child.state_done? || child.state_ready?)}
+        else
+          @state = :ready
+          state_changed
+        end
+      end
+    end
+
+    def state_changed(notify_only = false)
+      Dopi.log.debug("State of '#{name}' updated to #{@state.to_s}, notifying observers")
+      changed
+      notify_observers(notify_only)
+    end
+
+    def signals
+      @signals ||= Hash.new(false)
+    end
+
+    def reset_signals
+      @signals = Hash.new(false)
+      state_children.each {|child| child.reset_signals}
+    end
+
+    def signal_procs
+      @signal_procs ||= []
+    end
+
+    def on_signal(a_proc)
+      signal_procs << a_proc
+    end
+
+    def delete_on_signal(a_proc)
+      signal_procs.delete(a_proc)
+    end
+
+    def send_signal(signal)
+      unless signals[signal] == true
+        signal_procs.each {|p| p.call(signal)}
+        state_children.each {|child| child.send_signal(signal)}
+        signals[signal] = true
+      end
+    end
+
+  end
+end

data/lib/dopi/state_store.rb

@@ -0,0 +1,155 @@
+#
+# This is the DOPi state store which persists the state
+# of the steps between and during runs.
+#
+module Dopi
+  class StateStore
+
+    def initialize(plan_name, plan_store)
+      @plan_store  = plan_store
+      @plan_name   = plan_name
+      @state_store = @plan_store.state_store(plan_name, 'dopi')
+    end
+
+    def update(options = {})
+      if options[:clear]
+        clear(options)
+      elsif options[:ignore]
+        ignore(options)
+      else
+        update_state(options)
+      end
+    rescue DopCommon::UnknownVersionError => e
+      Dopi.log.warn("The state has an unknown plan version #{e.message}.")
+      Dopi.log.warn("Please update with the 'clear' or 'ignore' option")
+    rescue => e
+      Dopi.log.error("An error occured during update: #{e.message}")
+      Dopi.log.error("Please update with the 'clear' or 'ignore' option")
+    end
+
+    def persist_state(plan)
+      @state_store.transaction do
+        plan_state = plan.state_hash
+        Dopi.log.debug('Persisting plan state:')
+        Dopi.log.debug(plan_state)
+        @state_store[:state] = plan_state
+      end
+    end
+
+    def state_hash
+      @state_store.transaction(true) do
+        @state_store[:state] || {}
+      end
+    end
+
+    def method_missing(m, *args, &block)
+      @state_store.send(m, *args, &block)
+    end
+
+  private
+
+    def clear(options)
+      @state_store.transaction do
+        Dopi.log.debug("Clearing the state for plan #{@plan_name}")
+        ver = @plan_store.show_versions(@plan_name).last
+        plan = Dopi::Plan.new(@plan_store.get_plan(@plan_name))
+        @state_store[:state] = plan.state_hash
+        @state_store[:version] = ver
+      end
+    end
+
+    def ignore(options)
+      @state_store.transaction do
+        ver = @plan_store.show_versions(@plan_name).last
+        Dopi.log.debug("Ignoring update and setting state version of plan #{@plan_name} to #{ver}")
+        @state_store[:version] = ver
+      end
+    end
+
+    def update_state(options)
+      @state_store.update do |plan_diff|
+        Dopi.log.debug("Updating plan #{@plan_name}. This is the diff:")
+        Dopi.log.debug(plan_diff.to_s)
+
+        plan_diff.each do |patch|
+          match ||= update_rule_steps(patch)
+          unless match
+            Dopi.log.debug("No rule matched, ignoring patch: #{patch.to_s}")
+          end
+        end
+      end
+    end
+
+    def update_rule_steps(patch)
+      match = /^steps\.?(\w*)\[(\d+)\](.*)/.match(patch[1])
+      if match
+        step_set = match[1].empty? ? 'default' : match[1]
+        step_nr  = match[2].to_i
+        rest     = match[3]
+        unless update_rule_commands(patch, step_set, step_nr, rest)
+          case patch[0]
+          when '+' then add_step(step_set, step_nr)
+          when '-' then del_step(step_set, step_nr)
+          else
+            Dopi.log.debug("Step changed, ignoring patch: #{patch.to_s}")
+          end
+        end
+        return true
+      end
+      return false
+    end
+
+    def update_rule_commands(patch, step_set, step_nr, rest)
+      match = /^\.commands?\[(\d+)\](.*)/.match(rest)
+      if match
+        command_nr = match[1]
+        if /^\.verify_command/.match(rest)
+          Dopi.log.debug("Change in verify_command only, ignoring patch: #{patch.to_s}")
+        else
+          case patch[0]
+          when '+' then add_command(step_set, step_nr)
+          when '-' then del_command(step_set, step_nr)
+          else
+            Dopi.log.debug("Command changed, ignoring patch: #{patch.to_s}")
+          end
+        end
+        return true
+      end
+      return false
+    end
+
+    def add_step(step_set, step_nr)
+      @state_store[:state][:step_sets][step_set].insert(step_nr, {})
+    end
+
+    def del_step(step_set, step_nr)
+      @state_store[:state][:step_sets][step_set].delete_at(step_nr)
+    end
+
+    def add_command(step_set, step_nr, command_nr)
+      @state_store[:state][:step_sets][step_set][step_nr].each do |node|
+        node.insert(command_nr, {:command_state => :ready})
+      end
+    end
+
+    def del_command(step_set, step_nr, command_nr)
+      @state_store[:state][:step_sets][step_set][step_nr].each do |node|
+        node.delete_at(command_nr)
+      end
+    end
+
+  end
+
+  class StateStoreObserver
+
+    def initialize(plan, state_store)
+      @plan        = plan
+      @state_store = state_store
+    end
+
+    def update(notify_only = false)
+      @state_store.persist_state(@plan)
+    end
+
+  end
+end