dop_common 0.13.0

data/lib/dop_common/signal_handler.rb

@@ -0,0 +1,58 @@
+#
+# Defered signal handling
+#
+# This class will handle the trapping of signals
+#
+# This code uses tricks from http://timuruski.net/blog/2014/graceful-shutdown
+# for setup and teardown of the signal handlers. And the self pipe trick from
+# http://www.sitepoint.com/the-self-pipe-trick-explained
+#
+
+module DopCommon
+  class SignalHandler
+    DEFAULT_SIGNALS = [:INT, :QUIT, :TERM]
+
+    def initialize
+      @signal_queue = []
+      @self_reader, @self_writer = IO.pipe
+    end
+
+    def handle_signals(*signals)
+      signals = DEFAULT_SIGNALS if signals.empty?
+      old_handlers = setup_signal_traps(signals)
+      loop do
+        begin
+          if @signal_queue.any?
+            yield(@signal_queue.shift) if block_given?
+          else
+            IO.select([@self_reader])
+            @self_reader.read_nonblock(1)
+          end
+        rescue
+          break
+        end
+      end
+      teardown_signal_traps(old_handlers)
+    end
+
+    private
+
+    def setup_signal_traps(signals)
+      signals.each_with_object({}) do |signal, old_handlers|
+        DopCommon.log.debug("Installing trap for signal #{signal.to_s}")
+        old_handlers[signal] = Signal.trap(signal) do
+          @signal_queue << { signal => Time.now }
+          @self_writer.write_nonblock('.')
+        end
+      end
+    end
+
+    def teardown_signal_traps(old_handlers)
+      old_handlers.each do |signal, old_handler|
+        DopCommon.log.debug("Removing trap for signal #{signal.to_s}")
+        Signal.trap(signal, old_handler)
+      end
+    end
+
+  end
+end

data/lib/dop_common/state_store.rb

@@ -0,0 +1,95 @@
+#
+# This is a simple wrapper around YAML::Store to make versioning
+# and updating easier.
+#
+require 'yaml/store'
+require 'rb-inotify'
+
+module DopCommon
+  class UnknownVersionError < StandardError
+  end
+
+  class StateStore < YAML::Store
+
+    def initialize(state_file, plan_name, plan_cache)
+      @plan_name   = plan_name
+      @plan_cache  = plan_cache
+      @state_file  = state_file
+      @write_mutex = Mutex.new
+      super(@state_file)
+    end
+
+    # This is a wrapper around transaction to make sure we have a run lock.
+    # This will ensure that only ever one instance can write to this store.
+    def transaction(read_only = false, &block)
+      if read_only
+        super(read_only, &block)
+      else
+        @write_mutex.synchronize do
+          if @plan_cache.run_lock?(@plan_name)
+            # save the version on first write
+            super do
+              self[:version] = latest_version if self[:version].nil?
+            end
+            super(&block)
+          else
+            raise StandardError, "Not possible to write to #{@state_file} because we have no run lock"
+          end
+        end
+      end
+    end
+
+    def version
+      transaction(true) do
+        self[:version] || :new
+      end
+    end
+
+    def pending_updates?
+      case version
+      when :new, latest_version then false
+      else true
+      end
+    end
+
+    # update the state file. This takes a block which will receive a
+    # hash diff from the state version to the newest plan version.
+    # If the plan is new or already on the latest version the block will
+    # not be executed.
+    #
+    # The block is already inside a transaction. The version will be bumped
+    # to the latest version only if the transaction is successful.
+    def update
+      ver = version
+      return if ver == latest_version
+      return if ver == :new
+      raise UnknownVersionError.new(ver) unless version_exists?(ver)
+      DopCommon.log.info("Updating plan #{@plan_name} from version #{ver} to #{latest_version}")
+      transaction do
+        yield(@plan_cache.get_plan_hash_diff(@plan_name, ver, latest_version))
+        self[:version] = latest_version
+      end
+    end
+
+    # This method will take a block which will be executet every time the
+    # state file changes.
+    def on_change
+      notifier = INotify::Notifier.new
+      notifier.watch(@state_file, :modify) do
+        yield
+      end
+      notifier.run
+    end
+
+  private
+
+    def latest_version
+      @plan_cache.show_versions(@plan_name).last
+    end
+
+    def version_exists?(version)
+      @plan_cache.show_versions(@plan_name).include?(version)
+    end
+
+  end
+end

data/lib/dop_common/step.rb

@@ -0,0 +1,200 @@
+#
+# DOP common step hash parser
+#
+
+module DopCommon
+  class Step
+    include Validator
+    include RunOptions
+
+    def initialize(hash)
+      @hash = HashParser.symbolize_keys(hash)
+      HashParser.key_aliases(@hash, :commands, [:command])
+    end
+
+    def name
+      @name ||= @hash[:name] or
+        raise PlanParsingError, "Every step needs to have a 'name' key defined"
+    end
+
+    def validate
+      valitdate_shared_options
+      log_validation_method('name')
+      log_validation_method('nodes_valid?')
+      log_validation_method('exclude_nodes_valid?')
+      log_validation_method('nodes_by_config_valid?')
+      log_validation_method('exclude_nodes_by_config_valid?')
+      log_validation_method('roles_valid?')
+      log_validation_method('exclude_roles_valid?')
+      log_validation_method('commands_valid?')
+      r_name = @name || 'unknown' # name may not be set because of a previous error
+      try_validate_obj("Step #{r_name}: Can't validate the commands part because of a previous error"){commands}
+    end
+
+    def nodes
+      @nodes ||= nodes_valid? ?
+        HashParser.parse_pattern_list(@hash, :nodes) : []
+    end
+
+    def exclude_nodes
+      @exclude_nodes ||= exclude_nodes_valid? ?
+        HashParser.parse_pattern_list(@hash, :exclude_nodes) : []
+    end
+
+    def nodes_by_config
+      @nodes_by_config ||= nodes_by_config_valid? ?
+        HashParser.parse_hash_of_pattern_lists(@hash, :nodes_by_config) : {}
+    end
+
+    def exclude_nodes_by_config
+      @exclude_nodes_by_config ||= exclude_nodes_by_config_valid? ?
+        HashParser.parse_hash_of_pattern_lists(@hash, :exclude_nodes_by_config) : {}
+    end
+
+    def roles
+      @roles ||= roles_valid? ?
+        HashParser.parse_pattern_list(@hash, :roles) : []
+    end
+
+    def exclude_roles
+      @exclude_roles ||= exclude_roles_valid? ?
+        HashParser.parse_pattern_list(@hash, :exclude_roles) : []
+    end
+
+    def commands
+      @commands ||= commands_valid? ? create_commands : nil
+    end
+
+    def set_plugin_defaults
+      @set_plugin_defaults ||= set_plugin_defaults_valid? ?
+        parse_plugin_pattern_array(:set_plugin_defaults) : []
+    end
+
+    def delete_plugin_defaults
+      @delete_plugin_defaults ||= delete_plugin_defaults_valid? ?
+        parse_plugin_pattern_array(:delete_plugin_defaults) : []
+    end
+
+  private
+
+    def nodes_valid?
+      pattern_list_valid?(@hash, :nodes)
+    end
+
+    def exclude_nodes_valid?
+      pattern_list_valid?(@hash, :exclude_nodes)
+    end
+
+    def nodes_by_config_valid?
+      hash_of_pattern_lists_valid?(@hash, :nodes_by_config)
+    end
+
+    def exclude_nodes_by_config_valid?
+      hash_of_pattern_lists_valid?(@hash, :exclude_nodes_by_config)
+    end
+
+    def roles_valid?
+      pattern_list_valid?(@hash, :roles)
+    end
+
+    def exclude_roles_valid?
+      pattern_list_valid?(@hash, :exclude_roles)
+    end
+
+    def pattern_list_valid?(hash, key, optional = true)
+      HashParser.pattern_list_valid?(hash, key, optional)
+    rescue PlanParsingError => e
+      raise PlanParsingError, "Step #{@name}: #{e.message}"
+    end
+
+    def hash_of_pattern_lists_valid?(hash, key, optional = true)
+      HashParser.hash_of_pattern_lists_valid?(hash, key, optional)
+    rescue PlanParsingError => e
+      raise PlanParsingError, "Step #{@name}: #{e.message}"
+    end
+
+    def commands_valid?
+      @hash[:commands] or
+        raise PlanParsingError, "Step #{@name}: A commands key has to be defined"
+      case @hash[:commands]
+      when String, Hash
+        true
+      when Array
+        @hash[:commands].all?{|c| c.kind_of?(String) or c.kind_of?(Hash)} or
+          raise PlanParsingError, "Step #{@name}: All commands must be Strings or Hashes"
+      else
+        raise PlanParsingError,
+          "Step #{@name}: The value for commands has to be a string, a hash or an array"
+      end
+    end
+
+    def create_commands
+      [@hash[:commands]].flatten.map do |command|
+        ::DopCommon::Command.new(command)
+      end
+    end
+
+    def plugin_pattern_array_valid?(key)
+      @hash[key].all? do |entry|
+        entry.kind_of?(Hash) or
+          raise PlanParsingError, "Step #{@name}: Each entry in the '#{key}' array has to be a hash"
+        HashParser.key_aliases(entry, :plugins, ['plugins', :plugin, 'plugin'])
+        pattern_list_valid?(entry, :plugins, false)
+      end
+    end
+
+    def set_plugin_defaults_valid?
+      return false if @hash[:set_plugin_defaults].nil? # is optional
+      @hash[:set_plugin_defaults].kind_of?(Array) or
+        raise PlanParsingError, "Step #{@name}: The value of 'set_plugin_defaults' has to be an array"
+      plugin_pattern_array_valid?(:set_plugin_defaults)
+    end
+
+    def delete_plugin_defaults_valid?
+      return false if @hash[:delete_plugin_defaults].nil? # is optional
+      @hash[:delete_plugin_defaults].kind_of?(Array) or
+      @hash[:delete_plugin_defaults].kind_of?(String) or
+      @hash[:delete_plugin_defaults].kind_of?(Symbol) or
+        raise PlanParsingError, "Step #{@name}: The value of 'delete_plugin_defaults' has to be an array or :all"
+      unless @hash[:delete_plugin_defaults].kind_of?(Array)
+        ['all', 'All', 'ALL', :all].include?(@hash[:delete_plugin_defaults]) or
+          raise PlanParsingError, "Step #{@name}: The value of 'delete_plugin_defaults' has to be an array or :all"
+      else
+        return false unless plugin_pattern_array_valid?(:delete_plugin_defaults)
+        @hash[:delete_plugin_defaults].all? do |entry|
+          HashParser.key_aliases(entry, :delete_keys, ['delete_keys', :delete_key, 'delete_key'])
+          entry[:delete_keys].nil? and
+            raise PlanParsingError, "Step #{@name}: Each entry in the 'delete_plugin_defaults' array needs a valid value for 'delete_keys'"
+          entry[:delete_keys].kind_of?(Array) or
+          entry[:delete_keys].kind_of?(String) or
+          entry[:delete_keys].kind_of?(Symbol) or
+            raise PlanParsingError, "Step #{@name}: The value for 'delete_keys' in 'delete_plugin_defaults' has to be a string or an array"
+          if entry[:delete_keys].kind_of?(Array)
+            entry[:delete_keys].all?{|e| e.kind_of?(String) or e.kind_of?(Symbol)} or
+              raise PlanParsingError, "Step #{@name}: The elements in 'delete_keys' in 'delete_plugin_defaults' have to me strings or symbols"
+          end
+        end
+      end
+      true
+    end
+
+    def parse_plugin_pattern_array(key)
+      unless @hash[key].kind_of?(Array)
+        :all
+      else
+        @hash[key].map do |entry|
+          result = entry.dup
+          result[:plugins] = HashParser.parse_pattern_list(entry, :plugins)
+          delete_keys = case entry[:delete_keys]
+          when 'all', 'All', 'ALL', :all then :all
+          when String, Array then [entry[:delete_keys]].flatten
+          else nil
+          end
+          result[:delete_keys] = delete_keys if delete_keys
+          result
+        end
+      end
+    end
+
+  end
+end

data/lib/dop_common/step_set.rb

@@ -0,0 +1,41 @@
+#
+# DOP common step set hash parser
+#
+
+module DopCommon
+  class StepSet
+    include Validator
+
+    attr_reader :name
+
+    def initialize(name, steps_array)
+      @name = name
+      @steps_array = steps_array
+    end
+
+    def validate
+      log_validation_method(:steps_valid?)
+      try_validate_obj("StepSet #{name}: Can't validate the steps part because of a previous error"){steps}
+    end
+
+    def steps
+      @steps ||= steps_valid? ? create_steps : nil
+    end
+
+  private
+
+    def steps_valid?
+      @steps_array.any? or
+        raise PlanParsingError, "StepSet #{name}: no steps defined"
+      @steps_array.all?{|s| s.kind_of?(Hash)} or
+        raise PlanParsingError, "StepSet #{name}: steps array must only contain hashes"
+    end
+
+    def create_steps
+      @steps_array.map do |hash|
+        ::DopCommon::Step.new(hash)
+      end
+    end
+
+  end
+end

data/lib/dop_common/thread_context_logger.rb

@@ -0,0 +1,77 @@
+#
+# This is log formatter which will log to different files based on the context
+# which was set for the current thread. if no context was set then it will log
+# to the 'all' context.
+#
+# This is used to separate log for different nodes even for logs which are
+# generated in some external library which is not aware of the context.
+#
+module DopCommon
+  class ThreadContextLogger
+    def initialize(log_path, contexts, all = true)
+      @log_path = log_path
+      @contexts = contexts
+      @all      = all
+      @mutex    = Mutex.new
+      @loggers  = {}
+      @threads  = {}
+
+      FileUtils.mkdir_p(@log_path)
+      create
+    end
+
+    def create
+      @mutex.synchronize do
+        add('all') if @all
+        @contexts.each{|context| add(context)}
+      end
+    end
+
+    def cleanup
+      @mutex.synchronize do
+        @contexts.each{|context| remove(context)}
+      end
+    end
+
+    def log_context=(context)
+      @mutex.synchronize do
+        @threads[Thread.current.object_id.to_s] = context
+      end
+    end
+
+    private
+
+    def add(context)
+      log_file = File.join(@log_path, context)
+      logger = Logger.new(log_file)
+      if context == 'all'
+        logger.formatter = Logger::Formatter.new
+      else
+        logger.formatter = formatter(context)
+      end
+      logger.level = ::Logger.const_get(DopCommon.config.log_level.upcase)
+
+      @loggers[context] = logger
+      DopCommon.add_log_junction(logger)
+    end
+
+    def remove(context)
+      logger = @loggers[context]
+      DopCommon.remove_log_junction(logger)
+    end
+
+    def formatter(context)
+      orig_formatter = Logger::Formatter.new
+      Proc.new do |severity, datetime, progname, msg|
+        @mutex.synchronize do
+          if context == @threads[Thread.current.object_id.to_s]
+            orig_formatter.call(severity, datetime, progname, msg)
+          else
+            nil
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/dop_common/utils.rb

@@ -0,0 +1,106 @@
+module DopCommon
+  module Utils
+    def sanitize_env(additional_env_vars = {})
+      {
+        'HOME' => ENV['HOME'],
+        'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
+      }.merge(additional_env_vars)
+    end
+    module_function :sanitize_env
+
+    class DataSize
+      include Validator
+
+      KIBIBYTE = 1024.0
+      MEBIBYTE = 1048576.0
+      GIBIBYTE = 1073741824.0
+      KILOBYTE = 1000.0
+      MEGABYTE = 1000000.0
+      GIGABYTE = 1000000000.0
+
+      def initialize(input)
+        @input ||= input
+      end
+
+      def validate
+        log_validation_method(:input_valid?)
+      end
+
+      def size
+        @size ||= input_valid? ? create_size : nil
+      end
+      alias_method :bytes, :size
+      alias_method :b, :size
+
+      def kibibytes
+        size / KIBIBYTE
+      end
+      alias_method :k, :kibibytes
+
+      def mebibytes
+        size / MEBIBYTE
+      end
+      alias_method :m, :mebibytes
+
+      def gibibytes
+        size / GIBIBYTE
+      end
+      alias_method :g, :gibibytes
+
+      def kilobytes
+        size / KILOBYTE
+      end
+      alias_method :kb, :kilobytes
+
+      def megabytes
+        size / MEGABYTE
+      end
+      alias_method :mb, :megabytes
+
+      def gigabytes
+        size / GIGABYTE
+      end
+      alias_method :gb, :gigabytes
+
+      def to_s
+        size.to_s
+      end
+
+      private
+
+      def input_valid?
+        raise PlanParsingError, "DataSize: Invalid input '#{@input}'. It must be an integer or string" unless
+          [String, Fixnum].include?(@input.class)
+        raise PlanParsingError, "DataSize: Invalid input '#{@input}'. It must be greater than zero" if
+          @input.kind_of?(Fixnum) && @input < 1
+        raise PlanParsingError, "DataSize: Invalid input '#{@input}'. " \
+          "It must be a positive number followed by one of K,KB,M,MB,G,GB literals" if
+          @input.kind_of?(String) && @input !~  /^(([1-9]\d*)(\.\d+)?|0\.(0*[1-9]\d*))[KMG]B?$/
+        true
+      end
+
+      def create_size
+        if @input.kind_of?(String)
+          if @input.index(/K$/)
+            s = @input.sub(/K$/, '').to_f * KIBIBYTE
+          elsif @input.index(/M$/)
+            s = @input.sub(/M$/, '').to_f * MEBIBYTE
+          elsif @input.index(/G$/)
+            s = @input.sub(/G$/, '').to_f * GIBIBYTE
+          elsif @input.index(/KB$/)
+            s = @input.sub(/KB$/, '').to_f * KILOBYTE
+          elsif @input.index(/MB$/)
+            s = @input.sub(/MB$/, '').to_f * MEGABYTE
+          elsif @input.index(/GB$/)
+            s = @input.sub(/GB$/, '').to_f * GIGABYTE
+          else
+            s = @input
+          end
+          @size = s.to_i
+        else
+          @size = @input
+        end
+      end
+    end
+  end
+end

data/lib/dop_common/validator.rb

@@ -0,0 +1,53 @@
+#
+# DOP Common Validator
+#
+# Some Validation Helper stuff
+#
+
+module DopCommon
+  module Validator
+
+    def valid?
+      @validity = true
+      validate
+      @validity
+    end
+
+    def set_not_valid
+      @validity = false
+    end
+
+    def log_validation_method(method, error_klass = PlanParsingError)
+      begin
+        send(method)
+      rescue error_klass => e
+        set_not_valid
+        DopCommon.log.error(e.message)
+      end
+    end
+
+    def try_validate_obj(message, error_klass = PlanParsingError)
+      begin
+        obj = yield
+        if obj.kind_of?(Array)
+          obj.each do |x|
+            x.validate
+            set_not_valid unless x.valid?
+          end
+        elsif obj.kind_of?(Hash)
+          obj.each_value do |x|
+            x.validate
+            set_not_valid unless x.valid?
+          end
+        else
+          obj.validate
+          set_not_valid unless obj.valid?
+        end
+      rescue error_klass => e
+        set_not_valid
+        DopCommon.log.warn(message)
+      end
+    end
+
+  end
+end

data/lib/dop_common/version.rb

@@ -0,0 +1,3 @@
+module DopCommon
+  VERSION = "0.13.0"
+end

data/lib/dop_common.rb

@@ -0,0 +1,32 @@
+require "dop_common/version"
+require "dop_common/config"
+require "dop_common/log"
+require "dop_common/thread_context_logger"
+require "dop_common/validator"
+require "dop_common/hash_parser"
+require "dop_common/utils"
+require "dop_common/run_options"
+require "dop_common/plan"
+require 'dop_common/infrastructure'
+require 'dop_common/network'
+require 'dop_common/affinity_group'
+require 'dop_common/hooks'
+require 'dop_common/node'
+require 'dop_common/infrastructure_properties'
+require 'dop_common/interface'
+require 'dop_common/step'
+require 'dop_common/step_set'
+require 'dop_common/configuration'
+require 'dop_common/credential'
+require 'dop_common/command'
+require 'dop_common/pre_processor'
+require 'dop_common/plan_store'
+require 'dop_common/plan_cache'
+require 'dop_common/state_store'
+require 'dop_common/dns'
+require 'dop_common/data_disk'
+require 'dop_common/node_filter'
+require 'dop_common/signal_handler'
+
+module DopCommon
+end