dop_common 0.13.0

data/doc/plan_format_v0.0.2_snippets.md

@@ -0,0 +1,56 @@
+
+If you want to create a lot of nodes of the same kind it can be quite repetitive to write them all down individually. Instead you can create multiple nodes in one step:
+
+```yaml
+nodes:
+    mysql01.example.com:
+        …
+    web{i}.example.com:
+        range: 1..4
+        digits: 2
+        …
+```
+
+This will create a number of nodes in the given range. The string “{i}” will be replaced with the amount of digits given. The above example will create the following nodes:
+
+- mysql01.example.com
+- web01.example.com
+- web02.example.com
+- web03.example.com
+- web04.example.com
+
+
+
+----
+
+### Nodes
+
+the nodes configuration hash assigns variables to an individual node or to a selected set of nodes. Here are some examples:
+
+```yaml
+configuration:
+    nodes:
+        mysql01.example.com:
+            role: mysql
+            my_var:  my_value
+       '/web\d+\.example\.com/':
+            role: httpd_basic
+       web04.example.com:
+            role: httpd_special
+       '/haproxy\d+\.example\.com/':
+            role: haproxy
+       '/.example.com/'
+            my_other_var: my_other_value
+```
+
+You can assign different variables to the same node with different regular expressions, but you can not overwrite the values this way. dop_common will throw an error if two regular expression who match the same node set the same variable. On the other hand you can overwrite a value if you set it with a specific fqdn entry.
+
+The only relevant setting here for DOP is the role variable. The name of the role variable can be configured but defaults to 'role'. If hiera is configured and activated, then dop_common will take the role specified in hiera if found. The hierarchy for the role resolution is as follows:
+
+1. Hiera
+2. fqdn match in nodes configuration in Plan file
+3. Regular expression match in nodes configuration in Plan file
+4. Default
+
+A node always needs a role. If the parser finds no value for one of the specified nodes in the plan file and if no default is set, the dop_common will throw an error.
+

data/dop_common.gemspec

@@ -0,0 +1,44 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dop_common/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dop_common"
+  spec.version       = DopCommon::VERSION
+  spec.authors       = ["Andreas Zuber", "Pavol Dilung"]
+  spec.email         = ["zuber@puzzle.ch", "pavol.dilung@swisscom.com"]
+  spec.description   = <<-EOF
+    This gem is part of the Deployment and Orchestration for Puppet
+    or DOP for short. dop_common is a library for the parsing and
+    validation of DOP plan files.
+  EOF
+  spec.summary       = %q{DOP plan file parser and validation library}
+  spec.homepage      = ""
+  spec.license       = 'Apache-2.0'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 1.8.7'
+
+  spec.add_dependency 'hashdiff', '~> 0.3', '>= 0.3.1'
+  spec.add_dependency 'lockfile', '~> 2'
+  spec.add_dependency 'rb-inotify', '~> 0.9'
+  spec.add_dependency 'hiera', '~> 3'
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-legacy_formatters"
+  spec.add_development_dependency "rspec-mocks"
+  spec.add_development_dependency "rspec-collection_matchers"
+  # no ruby 1.8.7 support
+  #spec.add_development_dependency "rspec-command"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-doc"
+  spec.add_development_dependency "pry-byebug"
+end

data/lib/dop_common/affinity_group.rb

@@ -0,0 +1,57 @@
+#
+# DOP Common infrastructure hash parser
+#
+
+module DopCommon
+  class AffinityGroup
+    include Validator
+    include HashParser
+
+    attr_reader :name
+
+    def initialize(name, hash)
+      @name = name
+      @hash = symbolize_keys(hash)
+    end
+
+    def validate
+      log_validation_method(:positive_valid?)
+      log_validation_method(:enforce_valid?)
+      log_validation_method(:cluster_valid?)
+    end
+
+    def positive
+      @positive ||= positive_valid? ? @hash[:positive] : nil
+    end
+    alias_method :positive?, :positive
+
+    def enforce
+      @enforce ||= enforce_valid? ? @hash[:enforce] : nil
+    end
+    alias_method :enforced?, :enforce
+
+    def cluster
+      @ip_pool ||= cluster_valid? ? @hash[:cluster] : nil
+    end
+
+    private
+
+    def positive_valid?
+      raise PlanParsingError, "Affinity group #{@name}: positive flag must be one of 'true' or 'false'" unless
+        @hash[:positive].kind_of?(TrueClass) || @hash[:positive].kind_of?(FalseClass)
+      true
+    end
+
+    def enforce_valid?
+      raise PlanParsingError, "Affinity group #{@name}: enforce flag must be one of 'true' or 'false'" unless
+        @hash[:enforce].kind_of?(TrueClass) || @hash[:enforce].kind_of?(FalseClass)
+      true
+    end
+
+    def cluster_valid?
+      raise PlanParsingError, "Affinity group #{@name}: cluster must be a string" unless
+        @hash[:cluster].kind_of?(String)
+      true
+    end
+  end
+end

data/lib/dop_common/cli/global_options.rb

@@ -0,0 +1,37 @@
+#
+# DOPi CLI gloable options
+#
+
+module DopCommon
+  module Cli
+
+    def self.global_options(base)
+      base.class_eval do
+        desc 'Verbosity of the command line tool'
+        default_value 'INFO'
+        arg_name 'Verbosity'
+        flag [:verbosity, :v]
+
+        desc 'Show stacktrace on crash'
+        default_value DopCommon.config.trace
+        switch [:trace, :t]
+
+        desc 'Specify the directory where the plans and their state will be stored'
+        default_value DopCommon.config.plan_store_dir
+        arg_name 'DIR'
+        flag [:plan_store_dir, :s]
+
+        desc 'Directory for the log files'
+        default_value DopCommon.config.log_dir
+        arg_name 'LOGDIR'
+        flag [:log_dir]
+
+        desc 'Log level for the logfiles'
+        default_value DopCommon.config.log_level
+        arg_name 'LOGLEVEL'
+        flag [:log_level, :l]
+      end
+    end
+
+  end
+end

data/lib/dop_common/cli/log.rb

@@ -0,0 +1,51 @@
+require 'dop_common/log'
+
+module DopCommon
+  module Cli
+
+    # Default quiet log formatter for the CLI
+    class DefaultFormatter < Logger::Formatter
+      def call(severity, time, progname, msg)
+        "#{msg2str(msg)}\n"
+      end
+    end
+
+    # This adds code line context to the log output
+    class TraceFormatter < Logger::Formatter
+      def call(severity, datetime, progname, msg)
+        timestamp = datetime.strftime("%Y-%m-%d %H:%M:%S.%L")
+        hostname = ::Socket.gethostname.split('.').first
+        file, line, method = caller[11].sub(/.*\/([0-9a-zA-Z_\-.]+):(\d+):.+`(.+)'/, "\\1-\\2-\\3").gsub(":", " ").split("-")
+        pid = "[#{::Process.pid}]".ljust(7)
+        "#{timestamp} #{hostname} #{progname}#{pid} #{severity.ljust(5)} #{file}:#{line}:#{method}: #{msg2str(msg)}\n"
+      end
+    end
+
+    def self.initialize_logger(name, log_level, verbosity, trace)
+      # create a dummy logger and use the lowest log level configured
+      DopCommon.logger = Logger.new('/dev/null')
+      file_log_level = ::Logger.const_get(log_level.upcase)
+      cli_log_level = ::Logger.const_get(verbosity.upcase)
+      min_log_level = file_log_level < cli_log_level ? file_log_level : cli_log_level
+      DopCommon.log.level = min_log_level
+
+      # create the cli console logger
+      logger = Logger.new(STDOUT)
+      logger.level = cli_log_level
+      if trace
+        logger.formatter = DopCommon::Cli::TraceFormatter.new
+      else
+        logger.formatter = DopCommon::Cli::DefaultFormatter.new
+      end
+      DopCommon.add_log_junction(logger)
+
+      # create the cli file logger
+      FileUtils.mkdir_p(DopCommon.config.log_dir)
+      log_file = File.join(DopCommon.config.log_dir, name)
+      logger = Logger.new(log_file , 10, 1024000)
+      logger.level = ::Logger.const_get(DopCommon.config.log_level.upcase)
+      DopCommon.add_log_junction(logger)
+    end
+
+  end
+end

data/lib/dop_common/cli/node_selection.rb

@@ -0,0 +1,62 @@
+require 'json'
+
+module DopCommon
+  module Cli
+
+    def node_select_options(command)
+      command.desc 'Run plans for this nodes only'
+      command.default_value ""
+      command.arg_name 'node01.example.com,node02.example.com,/example\.com$/'
+      command.flag [:nodes]
+
+      command.desc 'Run plans for this roles only'
+      command.default_value ""
+      command.arg_name 'role01,role01,/^rolepattern/'
+      command.flag [:roles]
+
+      command.desc 'Exclude this nodes from the run'
+      command.default_value ""
+      command.arg_name 'node01.example.com,node02.example.com,/example\.com$/'
+      command.flag [:exclude_nodes]
+
+      command.desc 'Exclude this roles from the run'
+      command.default_value ""
+      command.arg_name 'role01,role01,/^rolepattern/'
+      command.flag [:exclude_roles]
+
+      command.desc 'Run plans for this nodes with this config only (You have to specify a JSON hash here)'
+      command.default_value "{}"
+      command.arg_name '\'{"var1": ["val1", "/val2/"], "var2": "val2"}\''
+      command.flag [:nodes_by_config]
+
+      command.desc 'Exclude nodes with this config from the run (You have to specify a JSON hash here)'
+      command.default_value "{}"
+      command.arg_name '\'{"var1": ["val1", "/val2/"], "var2": "val2"}\''
+      command.flag [:exclude_nodes_by_config]
+    end
+    module_function :node_select_options
+
+    def parse_node_select_options(options)
+      pattern_hash = {}
+      [:nodes, :roles, :exclude_nodes, :exclude_roles].each do |key|
+        hash = { key => options[key].split(',')}
+        pattern_hash[key] = DopCommon::HashParser.pattern_list_valid?(hash, key) ?
+          DopCommon::HashParser.parse_pattern_list(hash, key) : []
+      end
+      [:nodes_by_config, :exclude_nodes_by_config].each do |key|
+        hash = {key => JSON.parse(options[key])}
+        pattern_hash[key] = DopCommon::HashParser.hash_of_pattern_lists_valid?(hash, key) ?
+          DopCommon::HashParser.parse_hash_of_pattern_lists(hash, key) : {}
+      end
+      # Select all nodes if nothing is included
+      if [:nodes, :roles, :nodes_by_config].all?{|k| pattern_hash[k].empty?}
+        pattern_hash[:nodes] = :all
+      end
+      OpenStruct.new(pattern_hash)
+    rescue DopCommon::PlanParsingError => e
+      raise StandardError, "Error while parsing the node selection options: #{e.message}"
+    end
+    module_function :parse_node_select_options
+
+  end
+end

data/lib/dop_common/command.rb

@@ -0,0 +1,125 @@
+#
+# DOP Common command hash parser
+#
+
+module DopCommon
+  class Command
+    include Validator
+    include HashParser
+
+    attr_reader :hash
+
+    def initialize(hash)
+      @hash = symbolize_keys(hash)
+      @defaults = {
+        :plugin_timeout => 300,
+        :verify_after_run => false,
+      }
+    end
+
+    def validate
+      log_validation_method('plugin_valid?')
+      log_validation_method('verify_after_run_valid?')
+      if @hash.kind_of?(Hash)
+        log_validation_method('plugin_timeout_valid?')
+        log_validation_method('verify_commands_valid?')
+        r_plugin = @plugin || 'unknown' # name may not be set because of a previous error
+        try_validate_obj("Command #{r_plugin}: Can't validate the verify_commands part because of a previous error"){verify_commands}
+      end
+    end
+
+    def overwrite_defaults=(defaults_hash)
+      @defaults.merge!(defaults_hash)
+    end
+
+    # Add the plugin specific validator
+    def extended_validator=(obj)
+      @extended_validator = obj if obj.respond_to?(:validate)
+    end
+
+    def plugin
+      @plugin ||= plugin_valid? ? parse_plugin : nil
+    end
+
+    def title
+      @title ||= title_valid? ? @hash[:title] : plugin
+    end
+
+    def plugin_timeout
+      @plugin_timeout = plugin_timeout_valid? ? @hash[:plugin_timeout] : @defaults[:plugin_timeout]
+    end
+
+    def verify_commands
+      @verify_commands ||= verify_commands_valid? ? create_verify_commands : []
+    end
+
+    def verify_after_run
+      @verify_after_run = verify_after_run_valid? ? @hash[:verify_after_run] : @defaults[:verify_after_run]
+    end
+
+  private
+
+    def plugin_valid?
+      if @hash.kind_of?(String)
+        @hash.empty? and
+          raise PlanParsingError, "The value for 'command' can not be an empty string"
+      elsif @hash.kind_of?(Hash)
+        @hash.empty? and
+          raise PlanParsingError, "The value for 'command' can not be an empty hash"
+        @hash[:plugin] or
+          raise PlanParsingError, "The 'plugin key is missing in the 'command' hash"
+        @hash[:plugin].kind_of?(String) or
+          raise PlanParsingError, "The value for 'plugin' has to be a String with valid plugin name"
+        @hash[:plugin].empty? and
+          raise PlanParsingError, "The value for 'plugin' can not be an empty string"
+      else
+        raise PlanParsingError, "The value for 'command' must be a String with a plugin name or a hash"
+      end
+      true
+    end
+
+    def parse_plugin
+      case @hash
+        when String then @hash
+        when Hash   then @hash[:plugin]
+      end
+    end
+
+    def title_valid?
+      return false unless @hash.kind_of?(Hash)
+      return false if @hash[:title].nil?
+      @hash[:title].kind_of?(String) or
+        raise PlanParsingError, "The command title has to be a string"
+    end
+
+    def plugin_timeout_valid?
+      return false unless @hash.kind_of?(Hash)
+      return false if @hash[:plugin_timeout].nil? # plugin_timeout is optional
+      @hash[:plugin_timeout].kind_of?(Fixnum) or
+        raise PlanParsingError, "The value for 'plugin_timeout' has to be a number"
+    end
+
+    def verify_commands_valid?
+      return false unless @hash.kind_of?(Hash)
+      return false if @hash[:verify_commands].nil?
+      [Array, Hash, String].include? @hash[:verify_commands].class or
+        raise PlanParsingError, "The value for 'verify_commands' has to be a String, Hash or an Array"
+    end
+
+    def create_verify_commands
+      case @hash[:verify_commands]
+        when String, Hash then [ ::DopCommon::Command.new(@hash[:verify_commands]) ]
+        when Array then @hash[:verify_commands].map {|c| ::DopCommon::Command.new(c)}
+        else []
+      end
+    end
+
+    def verify_after_run_valid?
+      return false unless @hash.kind_of?(Hash)
+      return false if @hash[:verify_after_run].nil?
+      @hash[:verify_after_run].kind_of?(TrueClass) or @hash[:verify_after_run].kind_of?(FalseClass) or
+        raise PlanParsingError, "The value for 'verify_after_run' must be boolean"
+    end
+
+  end
+end

data/lib/dop_common/config/helper.rb

@@ -0,0 +1,39 @@
+#
+# Dop configurator helper functions
+#
+module DopCommon
+  class Config
+    module Helper
+      def self.included(base)
+        base.send(:extend, ClassMethods)
+      end
+
+      module ClassMethods
+        def user
+          Etc.getpwuid(Process.uid)
+        end
+
+        def is_root?
+          user.name == 'root'
+        end
+
+        def dop_home
+          File.join(user.dir, '.dop')
+        end
+
+        def conf_var(variable, options = {})
+          define_method(variable) do
+            unless instance_variable_defined?("@#{variable}")
+              default = default.call(self) if default.kind_of?(Proc)
+              instance_variable_set "@#{variable}", options[:default]
+            end
+            instance_variable_get "@#{variable}"
+          end
+          attr_writer variable
+        end
+      end
+
+    end
+  end
+end
+

data/lib/dop_common/config.rb

@@ -0,0 +1,66 @@
+#
+# Configration for DopCommon
+#
+# Configure the module in a block:
+#
+#   DopCommon.configure do |config|
+#     config.use_hiera = true
+#   end
+#
+require 'etc'
+require 'dop_common/config/helper'
+
+module DopCommon
+
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.configure
+    yield config
+  end
+
+  def self.configure=(options_hash)
+    options_hash.each do |key, value|
+      variable_name = '@' + key.to_s
+      if config.instance_variable_defined?( variable_name )
+        config.instance_variable_set( variable_name , value )
+      end
+    end
+  end
+
+  class Config
+    include DopCommon::Config::Helper
+
+		conf_var :trace,
+			default: false
+		conf_var :config_file,
+			default: is_root? ? '/etc/dop/dop.conf' : File.join(dop_home, 'dop.conf')
+		conf_var :plan_store_dir,
+			default: is_root? ? '/var/lib/dop/cache' : File.join(dop_home, 'cache')
+		conf_var :use_hiera,
+			default: true
+		conf_var :hiera_yaml,
+			default: is_root? ? '/etc/puppet/hiera.yaml' : File.join(user.dir, '.puppet', 'hiera.yaml')
+    conf_var :load_facts,
+      default: false
+    conf_var :facts_dir,
+      default: is_root? ? '/var/lib/puppet/yaml/facts' : File.join(user.dir, '.puppet', 'var', 'yaml', 'facts')
+		conf_var :role_variable,
+			default: 'role'
+		conf_var :role_default,
+			default: nil
+		conf_var :connection_check_timeout,
+			default: 5
+		conf_var :mco_config,
+			default: is_root? ? '/etc/mcollective/client.cfg' : File.join(user.dir, '.mcollective')
+		conf_var :mco_dopi_logger,
+			default: true
+		conf_var :log_dir,
+			default: is_root? ? '/var/log/dop/' : File.join(dop_home, 'log')
+    conf_var :log_level,
+      default: 'DEBUG'
+
+  end
+
+end

data/lib/dop_common/configuration.rb

@@ -0,0 +1,37 @@
+#
+# DOP Common configuration lookup
+#
+
+module DopCommon
+  class ConfigurationValueNotFound < StandardError
+  end
+
+  class Configuration
+
+    def initialize(hash)
+      @hash = hash
+    end
+
+    def lookup(source, key, scope)
+      element = traverse_hash(source)
+      if element.has_key?(key)
+        element[key]
+      else
+        raise DopCommon::ConfigurationValueNotFound
+      end
+    rescue => e
+      raise DopCommon::ConfigurationValueNotFound
+    end
+
+  private
+
+    def traverse_hash(source)
+      element = @hash
+      source.split('/').each do |level|
+        element = element[level]
+      end
+      element
+    end
+
+  end
+end