dopi 0.17.0

data/lib/dopi/step.rb

@@ -0,0 +1,227 @@
+#
+# Step
+#
+require 'parallel'
+
+module Dopi
+  class Step
+    include Dopi::State
+    include DopCommon::NodeFilter
+
+    DEFAULT_MAX_IN_FLIGHT = 3
+    DEFAULT_MAX_PER_ROLE  = -1
+
+    attr_accessor :plan, :nodes
+
+    def initialize(step_parser, plan)
+      @step_parser = step_parser
+      @plan        = plan
+      @nodes       = filter_nodes(plan.nodes, step_parser)
+
+      @next_mutex   = Mutex.new
+      @notify_mutex = Mutex.new
+      @queue        = Queue.new
+
+      command_sets.each{|command_set| state_add_child(command_set)}
+    end
+
+    # Loading queue object from yaml files results in not properly initialized
+    # queue and a type error when using it. Skip queue when converting to yaml.
+    # Will be nil after loading from yaml and must be re-created.
+    def to_yaml_properties
+      super - [:@queue]
+    end
+
+    def name
+      @step_parser.name
+    end
+
+    def valid?
+      if @nodes.empty?
+        Dopi.log.error("Step '#{name}': Nodes list is empty")
+        return false
+      end
+      # since they are identical in respect to parsing
+      # we only have to check one of them
+      command_sets.first.valid?
+    end
+
+    def command_sets
+      @command_sets ||= @nodes.map do |node|
+        delete_plugin_defaults
+        set_plugin_defaults
+        Dopi::CommandSet.new(@step_parser, self, node)
+      end
+    end
+
+    def run(run_options)
+      if state_done?
+        Dopi.log.info("Step '#{name}' is in state 'done'. Skipping")
+        return
+      end
+      Dopi.log.info("Starting to run step '#{name}'")
+
+      nodes_to_run = filter_nodes(@nodes, run_options[:run_for_nodes])
+      command_sets_to_run = command_sets.select {|cs| nodes_to_run.include?(cs.node)}
+
+      unless run_options[:noop]
+        run_canary(run_options, command_sets_to_run) if canary_host
+        run_command_sets(run_options, command_sets_to_run) unless state_failed?
+      else
+        command_sets_to_run.each{|command_set| command_set.run(run_options[:noop])}
+      end
+
+      Dopi.log.info("Step '#{name}' successfully finished.") if state_done?
+      Dopi.log.error("Step '#{name}' failed! Stopping execution.") if state_failed?
+    end
+
+    def load_state(state_hash)
+      command_sets.each do |command_set|
+        command_set_state = state_hash[command_set.name] || []
+        command_set.load_state(command_set_state)
+      end
+    end
+
+    def state_hash
+      command_sets_hash = {}
+      command_sets.each do |command_set|
+        command_sets_hash[command_set.name] = command_set.state_hash
+      end
+      command_sets_hash
+    end
+
+    private
+
+    def run_canary(run_options, command_sets_to_run)
+      pick = rand(command_sets_to_run.length - 1)
+      command_sets_to_run[pick].run(run_options[:noop])
+    end
+
+    def run_command_sets(run_options, command_sets_to_run)
+      in_threads = max_in_flight == -1 ? command_sets_to_run.length : max_in_flight
+      pick = lambda { next_command_set(command_sets_to_run) || Parallel::Stop }
+      Parallel.each(pick, :in_threads => in_threads) do |command_set|
+        plan.context_logger.log_context = command_set.node.name
+        command_set.run(run_options[:noop])
+        notify_done
+      end
+    end
+
+    # notify the waiting thread that a command_set has finished it's run
+    def notify_done
+      @notify_mutex.synchronize do
+        queue.push(1)
+      end
+    end
+
+    # This method returns the next command_set which is ready
+    # to run. If no node is ready because of constrains
+    # it will block the thread until notify_done was called
+    # from a finishing thread. If no command_set is in the state
+    # ready it will return nil.
+    def next_command_set(command_sets_to_run)
+      @next_mutex.synchronize do
+        ready_command_sets = command_sets_to_run.select{|n| n.state == :ready}
+        return nil if ready_command_sets.empty?
+        loop do
+          return nil if state_failed? or signals[:stop]
+          @notify_mutex.synchronize do
+            queue.clear
+            next_command_set = ready_command_sets.find{|cs| is_runnable?(cs.node)}
+            unless next_command_set.nil?
+              next_command_set.state_start
+              return next_command_set
+            end
+          end
+          queue.pop # wait until a thread notifies it has finished
+        end
+      end
+    end
+
+    # check if a node is runnable or if there are constrains
+    # which prevent it from running
+    def is_runnable?(node)
+      if max_per_role > 0
+        running_groups[node.role] < max_per_role
+      else
+        true
+      end
+    end
+
+    # return a hash with the group names as keys and the
+    # amount of running nodes as value
+    def running_groups
+      role_counter = Hash.new(0)
+      command_sets.each do |command_set|
+        if [:running, :starting].include? command_set.state
+          role_counter[command_set.node.role] += 1
+        end
+      end
+      role_counter
+    end
+
+    def max_in_flight
+      @max_in_flight ||= @step_parser.max_in_flight || @plan.max_in_flight || DEFAULT_MAX_IN_FLIGHT
+    end
+
+    def max_per_role
+      @max_per_role ||= @step_parser.max_per_role || @plan.max_per_role || DEFAULT_MAX_PER_ROLE
+    end
+
+    def canary_host
+      @canary_host ||= @step_parser.canary_host || @plan.canary_host
+    end
+
+    def delete_plugin_defaults
+      if @step_parser.delete_plugin_defaults == :all
+        # Wipe all the defaults
+        PluginManager.plugin_klass_list('^dopi/command/').each do |plugin_klass|
+          @nodes.each{|node| plugin_klass.delete_plugin_defaults(node.name)}
+        end
+      else
+        @step_parser.delete_plugin_defaults.each do |entry|
+          plugin_list(entry[:plugins]).each do |plugin_klass|
+            if entry[:delete_keys] == :all
+              @nodes.each{|node| plugin_klass.delete_plugin_defaults(node.name)}
+            else
+              entry[:delete_keys].each do |key|
+                @nodes.each{|node| plugin_klass.delete_plugin_default(node.name, key)}
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def set_plugin_defaults
+      @step_parser.set_plugin_defaults.each do |entry|
+        defaults_hash = entry.dup
+        defaults_hash.delete(:plugins)
+        plugin_list(entry[:plugins]).each do |plugin_klass|
+          @nodes.each{|node| plugin_klass.set_plugin_defaults(node.name, defaults_hash)}
+        end
+      end
+    end
+
+    def plugin_list(plugin_filter_list)
+      if plugin_filter_list == :all
+        PluginManager.plugin_klass_list('^dopi/command/')
+      else
+        all_plugin_names = PluginManager.plugin_name_list('^dopi/command/').map{|p| p.sub('dopi/command/', '')}
+        selected_plugin_names = plugin_filter_list.map do |filter|
+          case filter
+          when Regexp then all_plugin_names.select{|p| p =~ filter}
+          else all_plugin_names.select{|p| p == filter}
+          end
+        end
+        selected_plugin_names.flatten.uniq.map{|p| PluginManager.plugin_klass('dopi/command/' + p)}
+      end
+    end
+
+    # Will be skipped when dumping yaml, therefore nil after loading from yaml.
+    def queue
+      @queue ||= Queue.new
+    end
+
+  end
+end

data/lib/dopi/step_set.rb

@@ -0,0 +1,70 @@
+#
+# This class parses loades step sets
+#
+require 'yaml'
+require 'dop_common'
+
+module Dopi
+  class StepSet
+    include Dopi::State
+
+    def initialize(parsed_step_set, plan)
+      @parsed_step_set = parsed_step_set
+      @plan = plan
+      steps.each{|step| state_add_child(step)}
+    end
+
+    def name
+      @parsed_step_set.name
+    end
+
+    def run(run_options)
+      if state_done?
+        Dopi.log.info("Step set #{name} is in state 'done'. Nothing to do")
+        return
+      end
+      unless state_ready?
+        raise StandardError, "Step set #{name} is not in state 'ready'. Try to reset the plan"
+      end
+      steps.each do |step|
+        step.run(run_options)
+        break if signals[:stop] || state_failed?
+      end
+    end
+
+    # The main validation work is done in the dop_common
+    # parser. We just add the command plugin parsers
+    def valid?
+      validity = true
+      validity = false unless steps.all?{|step| step.valid? }
+      validity
+    rescue Dopi::NoRoleFoundError => e
+      Dopi.log.warn(e.message)
+    end
+
+    def steps
+      # Before all the new commands get parsed we have to make sure we
+      # Reset all the plugin defaults
+      PluginManager.plugin_klass_list('^dopi/command/').each do |plugin_klass|
+        plugin_klass.wipe_plugin_defaults
+      end
+      @steps ||= @parsed_step_set.steps.map do |parsed_step|
+        ::Dopi::Step.new(parsed_step, @plan)
+      end
+    end
+
+    def load_state(state_hash)
+      return if state_hash.empty?
+      steps.each_with_index do |step, i|
+        step.load_state(state_hash[i])
+      end
+    end
+
+    def state_hash
+      steps.map do |step|
+        step.state_hash
+      end
+    end
+
+  end
+end

data/lib/dopi/version.rb

@@ -0,0 +1,3 @@
+module Dopi
+  VERSION = '0.17.0'
+end

data/lib/dopi.rb

@@ -0,0 +1,165 @@
+require 'dop_common'
+require "dopi/error"
+require "dopi/log"
+require "dopi/pluginmanager"
+require "dopi/state"
+require "dopi/state_store"
+require "dopi/command_parser/exec"
+require "dopi/command_parser/env"
+require "dopi/command_parser/arguments"
+require "dopi/command_parser/credentials"
+require "dopi/command_parser/exit_code"
+require "dopi/command_parser/output"
+require "dopi/connector/local"
+require "dopi/connector/ssh"
+require "dopi/connector/winrm"
+require "dopi/command"
+require "dopi/command_set"
+require "dopi/node"
+require "dopi/plan"
+require "dopi/step"
+require "dopi/step_set"
+require "dopi/version"
+
+module Dopi
+
+  def self.valid?(raw_plan)
+    hash, _ = plan_store.read_plan_file(raw_plan)
+    plan_parser = DopCommon::Plan.new(hash)
+    plan = Dopi::Plan.new(plan_parser)
+    plan.valid?
+  end
+
+  def self.add(raw_plan)
+    raise StandardError, 'Plan not valid; did not add' unless valid?(raw_plan)
+    plan_store.add(raw_plan)
+  end
+
+  def self.update_plan(raw_plan, options = {})
+    raise StandardError, 'Plan not valid; did not add' unless valid?(raw_plan)
+    plan_name = plan_store.update(raw_plan)
+    update_state(plan_name, options)
+    plan_name
+  end
+
+  def self.update_state(plan_name, options = {})
+    plan_store.run_lock(plan_name) do
+      state_store = Dopi::StateStore.new(plan_name, plan_store)
+      state_store.update(options)
+    end
+  end
+
+  def self.remove(plan_name, remove_dopi_state = true, remove_dopv_state = false)
+    plan_store.remove(plan_name, remove_dopi_state, remove_dopv_state)
+  end
+
+  def self.list
+    plan_store.list
+  end
+
+  # TODO: this returns a plan with loaded state at the moment.
+  # THIS MAY BE CHANGED IN THE FUTURE!!
+  def self.show(plan_name)
+    ensure_plan_exists(plan_name)
+    state_store = Dopi::StateStore.new(plan_name, plan_store)
+    plan = get_plan(plan_name)
+    plan.load_state(state_store.state_hash)
+    plan
+  end
+
+  def self.run(plan_name, options = {})
+    ensure_plan_exists(plan_name)
+    update_state(plan_name)
+    plan_store.run_lock(plan_name) do
+      state_store = Dopi::StateStore.new(plan_name, plan_store)
+      dopv_state_store = plan_store.state_store(plan_name, 'dopv')
+      dopv_state_store.transaction(true) do
+        dopv_node_info = dopv_state_store[:nodes] || {}
+        api_node_info  = options[:node_info] || {}
+        options[:node_info] = dopv_node_info.merge(api_node_info)
+      end
+      plan = get_plan(plan_name)
+      plan.load_state(state_store.state_hash)
+      manager = nil
+      if block_given?
+        manager = Thread.new { yield(plan) }
+      else
+        run_signal_handler(plan)
+      end
+      begin
+        state_store_observer = Dopi::StateStoreObserver.new(plan, state_store)
+        plan.add_observer(state_store_observer)
+        plan.run(options)
+        manager.join if manager
+      ensure
+        state_store_observer.update
+      end
+    end
+  end
+
+  def self.reset(plan_name, force = false)
+    ensure_plan_exists(plan_name)
+    plan_store.run_lock(plan_name) do
+      state_store = Dopi::StateStore.new(plan_name, plan_store)
+      plan = get_plan(plan_name)
+      plan.load_state(state_store.state_hash)
+      plan.state_reset_with_children(force)
+      state_store.persist_state(plan)
+    end
+  end
+
+  def self.on_state_change(plan_name)
+    ensure_plan_exists(plan_name)
+    state_store = Dopi::StateStore.new(plan_name, plan_store)
+    state_store.on_change do
+      yield
+    end
+  end
+
+private
+
+  def self.plan_store
+    @plan_store ||= DopCommon::PlanStore.new(DopCommon.config.plan_store_dir)
+  end
+
+  def self.get_plan(plan_name)
+    raise StandardError, 'Please update the plan state, there are pending updates' if pending_updates?(plan_name)
+    plan_parser = plan_store.get_plan(plan_name)
+    Dopi::Plan.new(plan_parser)
+  end
+
+  def self.pending_updates?(plan_name)
+    state_store = Dopi::StateStore.new(plan_name, plan_store)
+    state_store.pending_updates?
+  end
+
+  def self.run_signal_handler(plan)
+    plan.reset_signals
+    signal_handler_thread = Thread.new do
+      Dopi.log.info("Starting signal handling")
+      signal_counter = 0
+      DopCommon::SignalHandler.new.handle_signals(:INT, :TERM) do
+        signal_counter += 1
+        case signal_counter
+        when 1
+          Dopi.log.warn("Signal received! The run will halt after all currently running commands are finished")
+          plan.send_signal(:stop)
+        when 2
+          Dopi.log.error("Signal received! Sending termination signal to all the processes!")
+          plan.send_signal(:abort)
+        when 3
+          Dopi.log.error("Signal received! Sending KILL signal to all the processes!")
+          plan.send_signal(:kill)
+        end
+      end
+    end
+    signal_handler_thread.abort_on_exception = true
+  end
+
+  def self.ensure_plan_exists(plan_name)
+    unless plan_store.list.include?(plan_name)
+      raise StandardError, "The plan #{plan_name} does not exist in the plan store"
+    end
+  end
+
+end

data/spec/command_helper.rb

@@ -0,0 +1,11 @@
+
+module CommandHelper
+
+  def create_command(hash)
+    node = instance_double('Dopi::Node', :name => 'test.example.com')
+    step = instance_double('Dopi::Step', :name => 'Fake step for tests')
+    command_parser = DopCommon::Command.new(hash)
+    Dopi::Command.create_plugin_instance(command_parser, step, node)
+  end
+
+end

data/spec/fixtures/mco_client.cfg

@@ -0,0 +1,26 @@
+main_collective = mcollective
+collectives = mcollective
+#libdir = /usr/libexec/mcollective
+logfile = /dev/null
+loglevel = info
+direct_addressing = 0
+
+# Plugins
+securityprovider = psk
+plugin.psk = vagrant
+
+connector = activemq
+
+plugin.activemq.pool.size = 1
+plugin.activemq.pool.1.host = 192.168.56.102
+plugin.activemq.pool.1.port = 61614
+plugin.activemq.pool.1.user = mcollective
+plugin.activemq.pool.1.password = vagrant
+
+plugin.activemq.pool.1.ssl = false
+
+# Facts
+#factsource = yaml
+#plugin.yaml = /etc/mcollective/facts.yaml
+
+#default_discovery_method = mc

data/spec/fixtures/plans/fail_on_timeout.yaml

@@ -0,0 +1,20 @@
+name: 'fail_on_timeout'
+
+infrastructures:
+  baremetal:
+    type: 'baremetal'
+
+nodes:
+  linux01.example.com:
+    infrastructure: 'baremetal'
+
+steps:
+  default:
+    - name: 'fail on timeout'
+      nodes: 'all'
+      max_per_role: 1
+      command:
+        - plugin: 'custom'
+          plugin_timeout: 1
+          exec: 'sleep 4'
+

data/spec/fixtures/plans/hello_world.yaml

@@ -0,0 +1,34 @@
+name: 'hello_world'
+max_in_flight: 1
+
+infrastructures:
+  'test':
+    type: 'baremetal'
+
+nodes:
+  'linux01.example.com':
+    infrastructure: 'test'
+
+credentials:
+  'linux_login':
+    type: 'username_password'
+    username: 'root'
+    password: 'puppet'
+
+steps:
+  - name: 'write hello world'
+    nodes: 'all'
+    command:
+      plugin: 'custom'
+      exec: 'echo'
+      arguments: '"hello world"'
+  - name: 'list the local dir'
+    nodes: 'all'
+    command:
+      plugin: 'custom'
+      exec: 'ls'
+  - name: 'list the environment'
+    nodes: 'all'
+    command:
+      plugin: 'custom'
+      exec: 'env'

data/spec/fixtures/plans/non_existing_node.yaml

@@ -0,0 +1,26 @@
+name: 'hello_world'
+max_in_flight: 1
+
+infrastructures:
+  'test':
+    type: 'baremetal'
+
+nodes:
+  'nonexisting01.example.com':
+    infrastructure: 'test'
+
+credentials:
+  'linux_login':
+    type: 'username_password'
+    username: 'root'
+    password: 'puppet'
+
+steps:
+  - name: 'Hello world on node'
+    nodes: 'all'
+    command:
+      plugin: 'custom'
+      credentials: 'linux_login'
+      exec: 'echo'
+      arguments: '"hello world"'
+

data/spec/fixtures/plans/test_role_variable.yaml

@@ -0,0 +1,29 @@
+name: 'test_role_variable'
+max_in_flight: 1
+
+infrastructures:
+  'test':
+    type: 'baremetal'
+
+nodes:
+  'linux01.example.com':
+    infrastructure: 'test'
+
+credentials:
+  'linux_login':
+    type: 'username_password'
+    username: 'root'
+    password: 'puppet'
+
+configuration:
+  nodes:
+    'linux01.example.com':
+      test_role: 'testnode'
+
+steps:
+  - name: 'write hello world'
+    roles: 'testnode'
+    command:
+      plugin: 'custom'
+      exec: 'echo'
+      arguments: '"hello world"'

data/spec/fixtures/puppet/Puppetfile

@@ -0,0 +1,8 @@
+forge "https://forgeapi.puppetlabs.com"
+
+mod 'theforeman-puppet'
+mod 'theforeman-foreman'
+mod 'puppetlabs-firewall'
+mod 'puppetlabs-java'
+mod 'jbussdieker-activemq'
+mod 'camptocamp-mcollective'

data/spec/fixtures/puppet/Puppetfile.lock

@@ -0,0 +1,57 @@
+FORGE
+  remote: https://forgeapi.puppetlabs.com
+  specs:
+    adrien-alternatives (0.3.0)
+    camptocamp-buildenv (0.3.17)
+    camptocamp-mcollective (3.1.3)
+      camptocamp-ruby (< 2.0.0, >= 1.0.0)
+      puppetlabs-concat (< 2.0.0, >= 1.0.0)
+      puppetlabs-stdlib (< 5.0.0, >= 3.2.0)
+    camptocamp-ruby (1.1.11)
+      camptocamp-buildenv (< 1.0.0, >= 0.0.2)
+      puppetlabs-stdlib (< 5.0.0, >= 3.2.0)
+    jbussdieker-activemq (0.0.3)
+    puppet-extlib (0.11.0)
+      puppetlabs-stdlib (>= 0)
+    puppetlabs-apache (1.8.1)
+      puppetlabs-concat (< 3.0.0, >= 1.1.1)
+      puppetlabs-stdlib (< 5.0.0, >= 2.4.0)
+    puppetlabs-apt (2.2.2)
+      puppetlabs-stdlib (< 5.0.0, >= 4.5.0)
+    puppetlabs-concat (1.2.5)
+      puppetlabs-stdlib (< 5.0.0, >= 3.2.0)
+    puppetlabs-firewall (1.8.0)
+    puppetlabs-java (1.4.3)
+      puppetlabs-stdlib (< 5.0.0, >= 2.4.0)
+    puppetlabs-postgresql (4.7.1)
+      puppetlabs-apt (< 3.0.0, >= 1.8.0)
+      puppetlabs-concat (< 3.0.0, >= 1.1.0)
+      puppetlabs-stdlib (~> 4.0)
+    puppetlabs-stdlib (4.11.0)
+    puppetlabs-xinetd (1.5.0)
+      puppetlabs-stdlib (>= 2.2.1)
+    theforeman-foreman (5.1.0)
+      adrien-alternatives (< 1.0.0, >= 0.3.0)
+      puppet-extlib (< 1.0.0, >= 0.10.4)
+      puppetlabs-apache (< 2.0.0, >= 1.2.0)
+      puppetlabs-apt (< 3.0.0)
+      puppetlabs-concat (< 3.0.0, >= 1.0.0)
+      puppetlabs-postgresql (< 5.0.0, >= 3.0.0)
+      puppetlabs-stdlib (< 5.0.0, >= 4.2.0)
+      theforeman-tftp (< 2.0.0, >= 1.4.0)
+    theforeman-puppet (4.3.1)
+      puppetlabs-apache (< 2.0.0, >= 1.2.0)
+      puppetlabs-concat (< 3.0.0, >= 1.0.0)
+      puppetlabs-stdlib (< 5.0.0, >= 4.2.0)
+    theforeman-tftp (1.7.0)
+      puppetlabs-stdlib (< 5.0.0, >= 2.3.0)
+      puppetlabs-xinetd (< 2.0.0, >= 1.1.0)
+
+DEPENDENCIES
+  camptocamp-mcollective (>= 0)
+  jbussdieker-activemq (>= 0)
+  puppetlabs-firewall (>= 0)
+  puppetlabs-java (>= 0)
+  theforeman-foreman (>= 0)
+  theforeman-puppet (>= 0)
+

data/spec/fixtures/puppet/hiera.yaml

@@ -0,0 +1,6 @@
+:backends:
+    - dop
+:hierarchy:
+    - nodes/%{::clientcert}
+    - roles/%{::role}
+    - defaults