dop_common 0.13.0

data/lib/dop_common/plan.rb

@@ -0,0 +1,188 @@
+#
+#
+#
+require 'yaml'
+
+module DopCommon
+  class PlanParsingError < StandardError
+  end
+
+  class Plan
+    include Validator
+    include HashParser
+    include RunOptions
+
+    def initialize(hash)
+      @hash = symbolize_keys(hash)
+    end
+
+    def validate
+      valitdate_shared_options
+      log_validation_method('name_valid?')
+      log_validation_method('infrastructures_valid?')
+      log_validation_method('nodes_valid?')
+      log_validation_method('step_sets_valid?')
+      log_validation_method('configuration_valid?')
+      log_validation_method('credentials_valid?')
+      log_validation_method(:hooks_valid?)
+      try_validate_obj("Plan: Can't validate the infrastructures part because of a previous error"){infrastructures}
+      try_validate_obj("Plan: Can't validate the nodes part because of a previous error"){nodes}
+      try_validate_obj("Plan: Can't validate the steps part because of a previous error"){step_sets}
+      try_validate_obj("Plan: Can't validate the credentials part because of a previous error"){credentials}
+      try_validate_obj("Infrastructure #{name}: Can't validate hooks part because of a previous error") { hooks }
+    end
+
+    def name
+      @name ||= name_valid? ?
+        @hash[:name] : Digest::SHA2.hexdigest(@hash.to_s)
+    end
+
+    def infrastructures
+      @infrastructures ||= infrastructures_valid? ?
+        create_infrastructures : nil
+    end
+
+    def nodes
+      @nodes ||= nodes_valid? ?
+        inflate_nodes : nil
+    end
+
+    def step_sets
+      @step_sets ||= step_sets_valid? ?
+        create_step_sets : []
+    end
+
+    def configuration
+      @configuration ||= configuration_valid? ?
+        DopCommon::Configuration.new(@hash[:configuration]) :
+        DopCommon::Configuration.new({})
+    end
+
+    def credentials
+      @credentials ||= credentials_valid? ?
+        create_credentials : {}
+    end
+
+    def find_node(name)
+      nodes.find{|node| node.name == name}
+    end
+
+    def hooks
+      @hooks ||= ::DopCommon::Hooks.new(hooks_valid? ? @hash[:hooks] : {})
+    end
+
+  private
+
+    def name_valid?
+      return false if @hash[:name].nil?
+      @hash[:name].kind_of?(String) or
+        raise PlanParsingError, 'The plan name has to be a String'
+      @hash[:name][/^[\w-]+$/,0] or
+        raise PlanParsingError, 'The plan name may only contain letters, numbers and underscores'
+    end
+
+    def infrastructures_valid?
+      @hash[:infrastructures] or
+        raise PlanParsingError, 'Plan: infrastructures hash is missing'
+      @hash[:infrastructures].kind_of?(Hash) or
+        raise PlanParsingError, 'Plan: infrastructures key has not a hash as value'
+      @hash[:infrastructures].any? or
+        raise PlanParsingError, 'Plan: infrastructures hash is empty'
+    end
+
+    def create_infrastructures
+      @hash[:infrastructures].map do |name, hash|
+        ::DopCommon::Infrastructure.new(name, hash, {:parsed_credentials => credentials})
+      end
+    end
+
+    def nodes_valid?
+      @hash[:nodes] or
+        raise PlanParsingError, 'Plan: nodes hash is missing'
+      @hash[:nodes].kind_of?(Hash) or
+        raise PlanParsingError, 'Plan: nodes key has not a hash as value'
+      @hash[:nodes].any? or
+        raise PlanParsingError, 'Plan: nodes hash is empty'
+      @hash[:nodes].values.all? { |n| n.kind_of?(Hash) } or
+        raise PlanParsingError, 'Plan: nodes must be of hash type'
+    end
+
+    def parsed_nodes
+      @parsed_nodes ||= @hash[:nodes].map do |name, hash|
+        ::DopCommon::Node.new(name.to_s, hash, {
+          :parsed_infrastructures => infrastructures,
+          :parsed_credentials     => credentials,
+          :parsed_hooks           => hooks,
+          :parsed_configuration   => configuration,
+        })
+      end
+    end
+
+    def inflate_nodes
+      parsed_nodes.map do |node|
+        node.inflatable? ? node.inflate : node
+      end.flatten
+    end
+
+    def step_sets_valid?
+      case @hash[:steps]
+      when nil then return false #steps can be nil for DOPv only plans
+      when Array then return true
+      when Hash # multiple step_sets defined
+        @hash[:steps].any? or
+          raise PlanParsingError, 'Plan: the hash in steps must not be empty'
+        @hash[:steps].keys.all?{|k| k.kind_of?(String)} or
+          raise PlanParsingError, 'Plan: all the keys in the steps hash have to be strings'
+        @hash[:steps].values.all?{|v| v.kind_of?(Array)} or
+          raise PlanParsingError, 'Plan: all values in the steps hash have to be arrays'
+      else
+        raise PlanParsingError, 'Plan: steps key has not a array or hash as value'
+      end
+      true
+    end
+
+    def create_step_sets
+      case @hash[:steps]
+      when Array
+        [::DopCommon::StepSet.new('default', @hash[:steps])]
+      when Hash
+        @hash[:steps].map do |name, steps|
+          ::DopCommon::StepSet.new(name, steps)
+        end
+      end
+    end
+
+    def configuration_valid?
+      return false if @hash[:configuration].nil? # configuration hash is optional
+      @hash[:configuration].kind_of? Hash or
+        raise PlanParsingError, "Plan: 'configuration' key has not a hash as value"
+    end
+
+    def credentials_valid?
+      return false if @hash[:credentials].nil? # credentials hash is optional
+      @hash[:credentials].kind_of? Hash or
+        raise PlanParsingError, "Plan: 'credentials' key has not a hash as value"
+      @hash[:credentials].keys.all?{|k| k.kind_of?(String) or k.kind_of?(Symbol)} or
+        raise PlanParsingError, "Plan: all keys in the 'credentials' hash have to be strings or symbols"
+      @hash[:credentials].values.all?{|v| v.kind_of?(Hash)} or
+        raise PlanParsingError, "Plan: all values in the 'credentials' hash have to be hashes"
+    end
+
+    def hooks_valid?
+      return false unless @hash.has_key?(:hooks)
+      raise PlanParsingError, "Plan: hooks, if specified, must be a non-empty hash" if
+        !@hash[:hooks].kind_of?(Hash) || @hash[:hooks].empty?
+      @hash[:hooks].keys.each do |h|
+        raise PlanParsingError, "Plan: invalid hook name '#{h}'" unless
+          h.to_s =~ /^(pre|post)_(create|update|destroy)_vm$/
+      end
+      true
+    end
+
+    def create_credentials
+      Hash[@hash[:credentials].map do |name, hash|
+        [name, ::DopCommon::Credential.new(name, hash)]
+      end]
+    end
+  end
+end

data/lib/dop_common/plan_cache.rb

@@ -0,0 +1,83 @@
+
+module DopCommon
+  class PlanCache
+
+    def initialize(plan_store)
+      @plan_store = plan_store
+      @plans = {}    # { plan_name => plan    }
+      @versions = {} # { plan      => version }
+      @nodes = {}    # { node_name => plan    }
+    end
+
+    # will return the plan of the node or nil
+    # if the node is not in a plan
+    def plan_by_node(node_name)
+      plan = @nodes[node_name]
+      if plan
+        refresh_plan(plan)
+        # this makes sure the node was not removed
+        plan = @nodes[node_name]
+        return plan if plan
+      end
+
+      refresh_all
+      return @nodes[node_name]
+    end
+
+  private
+
+    def refresh_plan(plan)
+      loaded_version = @versions[plan]
+      plan_name = plan.name
+      newest_version = @plan_store.show_versions(plan_name).last
+      unless loaded_version == newest_version
+        remove_plan(plan)
+        add_plan(plan_name)
+      end
+    rescue
+      remove_plan(plan)
+    end
+
+    def refresh_all
+      loaded_plan_names = @plans.keys
+      existing_plan_names = @plan_store.list
+      remove_old_plans(loaded_plan_names - existing_plan_names)
+      refresh_plans(existing_plan_names)
+    end
+
+    def refresh_plans(plan_names)
+      plan_names.each do |plan_name|
+        plan = @plans[plan_name]
+        if plan
+          refresh_plan(plan)
+        else
+          add_plan(plan_name)
+        end
+      end
+    end
+
+    def remove_old_plans(plan_names)
+      plan_names.each do |plan_name|
+        remove_plan(@plans[plan_name])
+      end
+    end
+
+    def remove_plan(plan_to_remove)
+      @nodes.delete_if{|node_name, plan| plan == plan_to_remove}
+      @plans.delete_if{|plan_name, plan| plan == plan_to_remove}
+      @versions.delete_if{|plan, version| plan == plan_to_remove}
+    end
+
+    def add_plan(plan_name)
+      version = @plan_store.show_versions(plan_name).last
+      plan    = @plan_store.get_plan(plan_name)
+      @plans[plan_name] = plan
+      @versions[plan] = version
+      plan.nodes.each do |node|
+        @nodes[node.name] = plan
+      end
+    end
+
+  end
+end
+

data/lib/dop_common/plan_store.rb

@@ -0,0 +1,263 @@
+#
+# DOP Common Plan Store
+#
+# This class will store validated and parsed plans and provides easy access to them
+#
+require 'yaml'
+require 'fileutils'
+require 'etc'
+require 'hashdiff'
+require 'lockfile'
+
+module DopCommon
+  class PlanExistsError < StandardError
+  end
+
+  class PlanStore
+
+    def initialize(plan_store_dir)
+      @plan_store_dir = plan_store_dir
+      @lockfiles = {}
+
+      # make sure the plan directory is created
+      FileUtils.mkdir_p(@plan_store_dir) unless File.directory?(@plan_store_dir)
+    end
+
+    # Add a new plan to the plan store
+    def add(raw_plan)
+      hash, yaml = read_plan_file(raw_plan)
+      plan = DopCommon::Plan.new(hash)
+
+      raise PlanExistsError, "There is already a plan with the name #{plan.name}" if plan_exists?(plan.name)
+      raise StandardError, 'Plan not valid. Unable to add' unless plan.valid?
+      raise StandardError, 'Some Nodes already exist. Unable to add' if node_duplicates?(plan)
+
+      versions_dir = File.join(@plan_store_dir, plan.name, 'versions')
+      FileUtils.mkdir_p(versions_dir) unless File.directory?(versions_dir)
+      run_lock(plan.name) do
+        save_plan_yaml(plan.name, yaml)
+      end
+
+      # make sure the state files are present
+      dopi_state = File.join(@plan_store_dir, plan.name, 'dopi.yaml')
+      dopv_state = File.join(@plan_store_dir, plan.name, 'dopv.yaml')
+      FileUtils.touch(dopi_state)
+      FileUtils.touch(dopv_state)
+
+      DopCommon.log.info("New plan #{plan.name} was added")
+      plan.name
+    end
+
+    # Update a plan already in the plan store
+    def update(raw_plan)
+      hash, yaml = read_plan_file(raw_plan)
+      plan = DopCommon::Plan.new(hash)
+
+      raise StandardError, "No plan with the name #{plan.name} found. Unable to update" unless plan_exists?(plan.name)
+      raise StandardError, 'Plan not valid. Unable to update' unless plan.valid?
+      raise StandardError, 'Some Nodes already exist in other plans. Unable to update' if node_duplicates?(plan)
+
+      run_lock(plan.name) do
+        save_plan_yaml(plan.name, yaml)
+      end
+      DopCommon.log.info("Plan #{plan.name} was updated")
+      plan.name
+    end
+
+    # remove a plan from the plan store
+    def remove(plan_name, remove_dopi_state = true, remove_dopv_state = false)
+      raise StandardError, "Plan #{plan_name} does not exist" unless plan_exists?(plan_name)
+      plan_dir = File.join(@plan_store_dir, plan_name)
+      versions_dir = File.join(plan_dir, 'versions')
+
+      # we have to remove the plan in two steps, so we don't
+      # delete the lockfile too soon.
+      run_lock(plan_name) do
+        FileUtils.remove_entry_secure(versions_dir)
+      end
+      info_file = File.join(plan_dir, 'run_lock_info')
+      FileUtils.remove_entry_secure(info_file)
+      if remove_dopi_state
+        dopi_state = File.join(plan_dir, 'dopi.yaml')
+        FileUtils.remove_entry_secure(dopi_state)
+      end
+      if remove_dopv_state
+        dopv_state = File.join(plan_dir, 'dopv.yaml')
+        FileUtils.remove_entry_secure(dopv_state)
+      end
+      if (Dir.entries(plan_dir) - [ '.', '..' ]).empty?
+        FileUtils.remove_entry_secure(plan_dir)
+      end
+      DopCommon.log.info("Plan #{plan_name} was removed")
+      plan_name
+    end
+
+    # return an array with all plan names in the plan store
+    def list
+      Dir.entries(@plan_store_dir).select do |entry|
+        versions_dir = File.join(@plan_store_dir, entry, "versions")
+        add_entry = true
+        add_entry = false if ['.', '..'].include?(entry)
+        add_entry = false if Dir[versions_dir + '/*.yaml'].empty?
+        add_entry
+      end
+    end
+
+    # returns a sorted array of versions for a plan (oldest version first)
+    def show_versions(plan_name)
+      raise StandardError, "Plan #{plan_name} does not exist" unless plan_exists?(plan_name)
+      versions_dir = File.join(@plan_store_dir, plan_name, 'versions')
+      Dir[versions_dir + '/*.yaml'].map {|yaml_file| File.basename(yaml_file, '.yaml')}.sort
+    end
+
+    # returns the yaml file content for the specified plan and version
+    # Returns the latest version if no version is specified
+    def get_plan_yaml(plan_name, version = :latest)
+      raise StandardError, "Plan #{plan_name} does not exist" unless plan_exists?(plan_name)
+
+      versions = show_versions(plan_name)
+      version = versions.last if version == :latest
+      raise StandardError, "Version #{version} of plan #{plan_name} not found" unless versions.include?(version)
+
+      yaml_file = File.join(@plan_store_dir, plan_name, 'versions', version + '.yaml')
+      File.read(yaml_file)
+    end
+
+    # return the hash for the plan in the store for a specific
+    # version. Returns the latest version if no version is specified
+    def get_plan_hash(plan_name, version = :latest)
+      yaml = get_plan_yaml(plan_name, version)
+      YAML.load(yaml)
+    end
+
+    # Get the plan object for the specified version directly
+    def get_plan(plan_name, version = :latest)
+      hash = get_plan_hash(plan_name, version)
+      DopCommon::Plan.new(hash)
+    end
+
+    def get_plan_hash_diff(plan_name, old_version, new_version = :latest)
+      old_hash = get_plan_hash(plan_name, old_version)
+      new_hash = get_plan_hash(plan_name, new_version)
+      HashDiff.best_diff(old_hash, new_hash)
+    end
+
+    # A run lock is used in all operations which change plans in the plan store.
+    def run_lock(plan_name)
+      remove_stale_lock(plan_name)
+      lockfile = run_lockfile(plan_name)
+      lockfile.lock
+      write_run_lock_info(plan_name, lockfile)
+      yield
+    rescue Lockfile::TimeoutLockError
+      raise StandardError, read_run_lock_info(plan_name)
+    ensure
+      lockfile.unlock if run_lock?(plan_name)
+    end
+
+    # return true if we have a run lock
+    def run_lock?(plan_name)
+      run_lockfile(plan_name).locked?
+    end
+
+    def state_store(plan_name, app_name)
+      state_file = File.join(@plan_store_dir, plan_name, app_name + '.yaml')
+      DopCommon::StateStore.new(state_file, plan_name, self)
+    end
+
+    # Returns true if a plan with that name already exists
+    # in the plan store.
+    def plan_exists?(plan_name)
+      versions_dir = File.join(@plan_store_dir, plan_name, 'versions')
+      Dir[versions_dir + '/*.yaml'].any?
+    end
+
+    # returns an array with [hash, yaml] of the plan. The plans should always be
+    # loaded with this method to make sure the plan is parsed with the
+    # pre_processor
+    def read_plan_file(raw_plan)
+      if raw_plan.kind_of?(Hash)
+        [raw_plan, raw_plan.to_yaml]
+      else
+        parsed_plan = PreProcessor.load_plan(raw_plan)
+        [YAML.load(parsed_plan), parsed_plan]
+      end
+    end
+
+    private
+
+    # returns true if a node in the plan is already present
+    # in an other plan already in the store.
+    def node_duplicates?(plan)
+      other_plans = list - [ plan.name ]
+      nodes = plan.nodes.map{|n| n.name}
+      other_plans.any? do |other_plan_name|
+        other_plan  = get_plan(other_plan_name)
+        other_nodes = other_plan.nodes.map{|n| n.name}
+        duplicates  = nodes & other_nodes
+        unless duplicates.empty?
+          DopCommon.log.error("Node(s) #{duplicates.join(', ')} already exist in plan #{other_plan_name}")
+          return true
+        end
+      end
+      false
+    end
+
+    # save a new version of the plan to the store
+    def save_plan_yaml(plan_name, yaml)
+      file_name = File.join(@plan_store_dir, plan_name, 'versions', new_version_string + '.yaml')
+      file = File.new(file_name, 'w')
+      file.write(yaml)
+      file.close
+    end
+
+    def new_version_string
+      time = Time.now.utc
+      usec = time.usec.to_s.rjust(6, '0')
+      time.strftime("%Y%m%d%H%M%S#{usec}")
+    end
+
+    def run_lockfile(plan_name)
+      lockfile = File.join(@plan_store_dir, plan_name, 'run_lock')
+      options = {:retry => 0, :timeout => 1, :max_age => 60}
+      @lockfiles[plan_name] ||= Lockfile.new(lockfile, options)
+    end
+
+    def stale_lock?(plan_name)
+      runlock_info = YAML.load(read_run_lock_info(plan_name))
+      pid = runlock_info['PID'].to_i
+      begin
+        Process.getpgid(pid)
+        false
+      rescue Errno::ESRCH
+        true
+      end
+    end
+
+    def remove_stale_lock(plan_name)
+      lockfile = run_lockfile(plan_name)
+      if File.exists?(lockfile.path) and stale_lock?(plan_name)
+        DopCommon.log.warn("Removing stale lockfile '#{lockfile.path}'")
+        File.delete(lockfile.path)
+      end
+    end
+
+    def write_run_lock_info(plan_name, lockfile)
+      info_file = File.join(@plan_store_dir, plan_name, 'run_lock_info')
+      user = Etc.getpwuid(Process.uid)
+      File.open(info_file, 'w') do |f|
+        f.puts "# A run lock for the plan #{plan_name} is in place!"
+        f.puts "Time: #{Time.now}"
+        f.puts "User: #{user.name}"
+        f.puts "PID: #{Process.pid}"
+        f.puts "Lockfile: #{lockfile.path}"
+      end
+    end
+
+    def read_run_lock_info(plan_name)
+      info_file = File.join(@plan_store_dir, plan_name, 'run_lock_info')
+      File.read(info_file)
+    end
+
+  end
+end

data/lib/dop_common/pre_processor.rb

@@ -0,0 +1,73 @@
+#
+# This is the plan preprocessor which merges the individual files
+# together.
+#
+require 'yaml'
+require 'pathname'
+
+module DopCommon
+  class PreProcessor
+
+    REGEXP = /(?:^| )include: (\S+)/
+
+    def self.load_plan(file)
+      file_abs = Pathname.new(file).expand_path.to_s
+      parse_file(file_abs, []).join
+    end
+
+  private
+
+    def self.parse_file(file, file_stack)
+      detect_loop(file_stack, file)
+      validate_file(file)
+      content = []
+      File.readlines(file).each_with_index do |line, i|
+        new_file_stack = file_stack.dup
+        new_file_stack << [file, i]
+        position = (line =~ REGEXP)
+        if position
+          position += 1 if position > 0
+          filtered_name = filter_name(line[REGEXP, 1])
+          new_file = absolute_filepath(file, filtered_name)
+          spacer = ' ' * position
+          content += parse_file(new_file, new_file_stack).map {|l| spacer + l}
+        else
+          content << line
+        end
+      end
+      content
+    end
+
+    def self.absolute_filepath(file, new_file)
+      if Pathname.new(new_file).absolute?
+        new_file
+      else
+        base_dir = Pathname.new(file).expand_path.dirname
+        File.join(base_dir, new_file)
+      end
+    end
+
+    def self.filter_name(file)
+      file[/^"(.*)"$/, 1] or
+      file[/^'(.*)'$/, 1] or
+      file
+    end
+
+    def self.validate_file(file)
+      File.exist?(file) or
+        raise PlanParsingError, "PreProcessor: The included file #{file} does not exist!"
+      File.readable?(file) or
+        raise PlanParsingError, "PreProcessor: The included file #{file} is not readable!"
+    end
+
+    def self.detect_loop(file_stack, file)
+      files = file_stack.map{|f| f[0] }
+      if files.include?(file)
+        files_i = file_stack.map{|f| "#{f[0]}:#{f[1]}" }
+        files_i << file
+        raise PlanParsingError, "PreProcessor: Include loop detected [ #{files_i.join(' --> ')} ]"
+      end
+    end
+
+  end
+end

data/lib/dop_common/run_options.rb

@@ -0,0 +1,56 @@
+#
+# Parsing of values that can be set on multiple levels
+#
+require 'yaml'
+
+module DopCommon
+  module RunOptions
+    include Validator
+
+    def valitdate_shared_options
+      log_validation_method('max_in_flight_valid?')
+      log_validation_method('max_per_role_valid?')
+      log_validation_method('canary_host_valid?')
+    end
+
+    def max_in_flight
+      @max_in_flight ||= max_in_flight_valid? ?
+        @hash[:max_in_flight] : nil
+    end
+
+    def max_per_role
+      @max_per_role ||= max_per_role_valid? ?
+        @hash[:max_per_role] : nil
+    end
+
+    def canary_host
+      @canary_host ||= canary_host_valid? ?
+        @hash[:canary_host] : false
+    end
+
+  private
+
+    def max_in_flight_valid?
+      return false if @hash[:max_in_flight].nil? # max_in_flight is optional
+      @hash[:max_in_flight].kind_of?(Fixnum) or
+        raise PlanParsingError, 'Plan: max_in_flight has to be a number'
+      @hash[:max_in_flight] >= -1 or
+        raise PlanParsingError, 'Plan: max_in_flight has to be greater than -1'
+    end
+
+    def max_per_role_valid?
+      return false if @hash[:max_per_role].nil? # max_per_role is optional
+      @hash[:max_per_role].kind_of?(Fixnum) or
+        raise PlanParsingError, 'Plan: max_per_role has to be a number'
+      @hash[:max_per_role] >= -1 or
+        raise PlanParsingError, 'Plan: max_per_role has to be greater than -1'
+    end
+
+    def canary_host_valid?
+      return false if @hash[:canary_host].nil?
+      @hash[:canary_host].kind_of?(TrueClass) or @hash[:canary_host].kind_of?(FalseClass) or
+        raise PlanParsingError, "Step #{@name}: The value for canary_host must be boolean"
+    end
+
+  end
+end