corl 0.4.0

data/lib/corl_core/util/ssh.rb

@@ -0,0 +1,286 @@
+
+module CORL
+module Util
+class SSH < Core
+  
+  #-----------------------------------------------------------------------------
+  # User key home
+  
+  @@key_path = nil
+  
+  #---
+  
+  def self.key_path
+    unless @@key_path
+      home_path  = ( ENV['USER'] == 'root' ? '/root' : ENV['HOME'] ) # In case we are using sudo
+      @@key_path = File.join(home_path, '.ssh')
+    
+      FileUtils.mkdir(@@key_path) unless File.directory?(@@key_path)
+    end
+    @@key_path
+  end
+  
+  #-----------------------------------------------------------------------------
+  # Instance generators
+  
+  def self.generate(options = {})
+    config = Config.ensure(options)
+    
+    private_key  = config.get(:private_key, nil)
+    original_key = nil
+    key_comment  = config.get(:comment, '')    
+    
+    if private_key.nil?
+      key_type    = config.get(:type, "RSA")
+      key_bits    = config.get(:bits, 2048)
+      passphrase  = config.get(:passphrase, nil)
+    
+      key_data = SSHKey.generate(
+        :type       => key_type, 
+        :bits       => key_bits, 
+        :comment    => key_comment, 
+        :passphrase => passphrase
+      )
+      is_new = true
+      
+    else
+      if private_key.include?('PRIVATE KEY')
+        original_key = private_key   
+      else
+        original_key = Disk.read(private_key)
+      end
+      
+      key_data = SSHKey.new(original_key, :comment => key_comment) if original_key
+      is_new   = false
+    end
+    
+    return nil unless key_data && ! key_data.ssh_public_key.empty?
+    Keypair.new(key_data, is_new, original_key)
+  end
+  
+  #-----------------------------------------------------------------------------
+  # Checks
+  
+  def self.valid?(public_ssh_key)
+    SSHKey.valid_ssh_public_key?(public_ssh_key)
+  end
+  
+  #-----------------------------------------------------------------------------
+  # Keypair interface
+    
+  class Keypair
+    attr_reader :type, :private_key, :encrypted_key, :public_key, :ssh_key
+    
+    def initialize(key_data, is_new, original_key)
+      @type          = key_data.type
+      @private_key   = key_data.private_key
+      @encrypted_key = is_new ? key_data.encrypted_private_key : original_key
+      @public_key    = key_data.public_key
+      @ssh_key       = key_data.ssh_public_key
+    end
+    
+    #---
+    
+    def store(key_path = nil, key_base = 'id')
+      key_path         = SSH.key_path if key_path.nil?
+      private_key_file = File.join(key_path, "#{key_base}_#{type.downcase}")
+      public_key_file  = File.join(key_path, "#{key_base}_#{type.downcase}.pub")
+      
+      private_success = Disk.write(private_key_file, encrypted_key)
+      FileUtils.chmod(0600, private_key_file) if private_success
+      
+      public_success  = Disk.write(public_key_file, ssh_key)
+      
+      if private_success && public_success
+        return { :private_key => private_key_file, :public_key => public_key_file }
+      end
+      false
+    end
+  end
+  
+  #-----------------------------------------------------------------------------
+  # SSH Execution interface
+  
+  @@sessions = {}
+  
+  #---
+  
+  def self.session_id(hostname, user)
+    "#{hostname}-#{user}"  
+  end
+  
+  #---
+  
+  def self.session(hostname, user, port = 22, private_key = nil, reset = false, options = {})
+    require 'net/ssh'
+    
+    ssh_options = Config.new({
+      :user_known_hosts_file => [ File.join(key_path, 'known_hosts'), File.join(key_path, 'known_hosts2') ],
+      :key_data              => [],
+      :keys_only             => false,
+      :auth_methods          => [ 'publickey' ],
+      :paranoid              => :very
+    }).import(options)
+    
+    ssh_options[:port] = port
+    ssh_options[:keys] = private_key.nil? ? [] : [ private_key ]
+    
+    session_id = session_id(hostname, user)
+    
+    if reset || ! @@sessions.has_key?(session_id)
+      @@sessions[session_id] = Net::SSH.start(hostname, user, ssh_options.export)
+    end
+    yield(@@sessions[session_id]) if block_given? && @@sessions[session_id]
+    @@sessions[session_id] 
+  end
+  
+  def self.init_session(hostname, user, port = 22, private_key = nil, options = {})
+    session(hostname, user, port, private_key, true, options)  
+  end
+  
+  #---
+  
+  def self.close(hostname = nil, user = nil)
+    if hostname && user.nil? # Assume we entered a session id
+      if @@sessions.has_key?(hostname)
+        @@sessions[hostname].close
+        @@sessions.delete(hostname)  
+      end
+      
+    elsif hostname && user # Generate session id from args
+      session_id = session_id(hostname, user)
+      
+      if @@sessions.has_key?(session_id)
+        @@sessions[session_id].close
+        @@sessions.delete(session_id)  
+      end
+      
+    else # Close all connections
+      @@sessions.keys.each do |id|
+        @@sessions[id].close
+        @@sessions.delete(id)      
+      end
+    end
+  end
+  
+  #---
+  
+  def self.exec(hostname, user, commands)
+    results = []
+        
+    begin
+      session(hostname, user) do |ssh|
+        Data.array(commands).each do |command|
+          command = command.flatten.join(' ') if command.is_a?(Array)
+          command = command.to_s
+          result  = Shell::Result.new(command)
+              
+          ssh.open_channel do |ssh_channel|
+            ssh_channel.request_pty
+            ssh_channel.exec(command) do |channel, success|
+              unless success
+                raise "Could not execute command: #{command.inspect}"
+              end
+
+              channel.on_data do |ch, data|
+                result.append_output(data)
+                yield(:output, command, data) if block_given?
+              end
+
+              channel.on_extended_data do |ch, type, data|
+                next unless type == 1
+                result.append_errors(data)
+                yield(:error, command, data) if block_given?
+              end
+
+              channel.on_request('exit-status') do |ch, data|
+                result.status = data.read_long
+              end
+
+              channel.on_request('exit-signal') do |ch, data|
+                result.status = 255
+              end
+            end
+          end
+          ssh.loop              
+          results << result
+        end
+      end
+    rescue Net::SSH::HostKeyMismatch => error
+      error.remember_host!
+      sleep 0.2
+      retry
+    end
+    results  
+  end
+  
+  #---
+  
+  def self.download(hostname, user, remote_path, local_path, options = {})
+    config = Config.ensure(options)
+    
+    require 'net/scp'
+    
+    # Accepted options:
+    # * :recursive - the +remote+ parameter refers to a remote directory, which
+    # should be downloaded to a new directory named +local+ on the local
+    # machine.
+    # * :preserve - the atime and mtime of the file should be preserved.
+    # * :verbose - the process should result in verbose output on the server
+    # end (useful for debugging).
+    #
+    config.init(:recursive, true)
+    config.init(:preserve, true)
+    config.init(:verbose, true)
+    
+    blocking = config.delete(:blocking, true)
+    
+    session(hostname, user) do |ssh|
+      if blocking
+        ssh.scp.download!(remote_path, local_path, config.export) do |ch, name, received, total|
+          yield(name, received, total) if block_given?
+        end
+      else
+        ssh.scp.download(remote_path, local_path, config.export)
+      end
+    end
+  end
+  
+  #---
+  
+  def self.upload(hostname, user, local_path, remote_path, options = {})
+    config = Config.ensure(options)
+    
+    require 'net/scp'
+    
+    # Accepted options:
+    # * :recursive - the +local+ parameter refers to a local directory, which
+    # should be uploaded to a new directory named +remote+ on the remote
+    # server.
+    # * :preserve - the atime and mtime of the file should be preserved.
+    # * :verbose - the process should result in verbose output on the server
+    # end (useful for debugging).
+    # * :chunk_size - the size of each "chunk" that should be sent. Defaults
+    # to 2048. Changing this value may improve throughput at the expense
+    # of decreasing interactivity.
+    #
+    config.init(:recursive, true)
+    config.init(:preserve, true)
+    config.init(:verbose, true)
+    config.init(:chunk_size, 2048)
+    
+    blocking = config.delete(:blocking, true)
+    
+    session(hostname, user) do |ssh|
+      if blocking
+        ssh.scp.upload!(local_path, remote_path, config.export) do |ch, name, sent, total|
+          yield(name, sent, total) if block_given?
+        end
+      else
+        ssh.scp.upload(local_path, remote_path, config.export)
+      end
+    end
+  end
+end
+end
+end

data/lib/facter/corl_config_ready.rb

@@ -0,0 +1,13 @@
+
+Facter.add(:corl_config_ready) do  
+  setcode do
+    begin 
+      require 'corl'      
+      configured = CORL::Config.config_initialized?
+      
+    rescue Exception # Prevent abortions.
+    end
+  
+    configured ? true : nil
+  end
+end

data/lib/facter/corl_exists.rb

@@ -0,0 +1,15 @@
+
+Facter.add(:corl_exists) do
+  confine :kernel => :linux
+  
+  setcode do
+    begin
+      Facter::Util::Resolution::exec('gem list corl -i 2> /dev/null')
+      corl_exists = true if $?.exitstatus == 0
+      
+    rescue Exception # Prevent abortions.
+    end
+    
+    corl_exists ? true : nil
+  end
+end

data/lib/facter/corl_network.rb

@@ -0,0 +1,17 @@
+
+Facter.add(:corl_network) do
+  confine :kernel => :linux
+    
+  setcode do
+    require 'corl'
+    
+    network_path = '/var/corl'
+    
+    CORL.exec(:network_location) do |op, results|
+      if op == :process
+        network_path = results unless results.nil? || ! File.directory?(results) 
+      end
+    end    
+    network_path
+  end
+end

data/lib/hiera/corl_logger.rb

@@ -0,0 +1,18 @@
+
+class Hiera
+  module CORL_logger
+    class << self
+      def suitable?
+        defined?(::CORL) == "constant"
+      end
+
+      def warn(message)
+        ::CORL.logger.warn("hiera: #{message}")
+      end
+
+      def debug(message)
+        ::CORL.logger.debug("hiera: #{message}")
+      end
+    end
+  end
+end

data/lib/puppet/indirector/corl.rb

@@ -0,0 +1,27 @@
+
+require 'puppet/indirector/terminus'
+
+class Puppet::Indirector::CORL < Puppet::Indirector::Terminus
+  
+  def initialize(*args)
+    unless CORL::Config.initialized?
+      raise "CORL terminus not supported without the CORL library"
+    end
+    super
+  end
+  
+  #---
+
+  def find(request)
+    config = CORL::Config.init({}, [ :all, :param, :data_binding ], {
+      :hiera_scope  => request.options[:variables],
+      :puppet_scope => request.options[:variables],
+      :search       => 'core::default',
+      :search_name  => false,
+      :init_fact    => 'corl_config_ready',
+      :force        => true,
+      :merge        => true
+    })    
+    value = CORL::Config.lookup(request.key, nil, config)
+  end
+end

data/lib/puppet/indirector/data_binding/corl.rb

@@ -0,0 +1,6 @@
+
+require 'puppet/indirector/corl'
+
+class Puppet::DataBinding::CORL < Puppet::Indirector::CORL
+  desc "Retrieve data using CORL lookup methods."
+end

data/lib/puppet/parser/functions/config_initialized.rb

@@ -0,0 +1,26 @@
+#
+# config_initialized.rb
+#
+# This function checks if the configuration system is fully configured 
+# and ready to query.
+#
+module Puppet::Parser::Functions
+  newfunction(:config_initialized, :type => :rvalue, :doc => <<-EOS
+This function checks if Hiera is fully configured and ready to query.
+    EOS
+) do |args|
+    
+    value = nil
+    CORL.run do
+      options = ( args[0].is_a?(Hash) ? args[0] : {} )
+    
+      config = CORL::Config.init_flat(options, [ :init, :config_initialized ], {
+        :hiera_scope  => self,
+        :puppet_scope => self,
+        :init_fact    => 'corl_config_ready'
+      })        
+      value = CORL::Config.initialized?(config)
+    end
+    return value
+  end
+end

data/lib/puppet/parser/functions/corl_include.rb

@@ -0,0 +1,44 @@
+#
+# corl_include.rb
+#
+# This function includes classes based on dynamic configurations.
+# following this order
+# - Hiera backend, if present (no prefix)
+# - ::data::default::varname
+# - ::varname
+# - {default parameter}
+#
+module Puppet::Parser::Functions
+  newfunction(:corl_include, :doc => <<-EOS
+This function performs a lookup for a variable value in various locations following this order:
+- Hiera backend, if present (no prefix)
+- ::data::default::varname
+- ::varname
+- {default parameter}
+If no value is found in the defined sources, it does not include any classes.
+    EOS
+) do |args|
+    
+    CORL.run do
+      raise(Puppet::ParseError, "corl_include(): Define at least the variable name " +
+        "given (#{args.size} for 1)") if args.size < 1
+
+      var_name   = args[0]
+      parameters = ( args.size > 1 ? args[1] : {} )
+      options    = ( args.size > 2 ? args[2] : {} ) 
+            
+      unless CORL.provisioner(:puppet).include(var_name, parameters, options)
+        # Throw an error if we didn't evaluate all of the classes.
+        str = "Could not find class"
+        str += "es" if missing.length > 1
+
+        str += " " + missing.join(", ")
+
+        if n = namespaces and ! n.empty? and n != [""]
+          str += " in namespaces #{@namespaces.join(", ")}"
+        end
+        self.fail Puppet::ParseError, str
+      end      
+    end
+  end
+end

data/lib/puppet/parser/functions/corl_resources.rb

@@ -0,0 +1,58 @@
+#
+# corl_resources.rb
+#
+# This function adds resource definitions of a specific type to the Puppet catalog
+# - Requires
+# - -> Puppet resource definition name (define)
+# - -> Hiera lookup name (full name)
+# - Optional
+# - -> default values for new resources
+# If no resources are found, it returns without creating anything.
+#
+module Puppet::Parser::Functions  
+  newfunction(:corl_resources, :doc => <<-EOS
+This function adds resource definitions of a specific type to the Puppet catalog
+- Requires
+- -> Puppet resource definition name (define)
+- -> Hiera lookup name (full name)
+- Optional
+- -> default values for new resources
+If no resources are found, it returns without creating anything.
+    EOS
+) do |args|
+    
+    CORL.run do
+      raise(Puppet::ParseError, "corl_resources(): Define at least the resource type and optional variable name " +
+        "given (#{args.size} for 1)") if args.size < 1
+      
+      definition_name = args[0]
+      type_name       = definition_name.sub(/^\@?\@/, '')
+      
+      resources       = ( args[1] ? args[1] : definition_name )
+      defaults        = ( args[2] ? args[2] : {} )    
+      
+      tag             = ( args[3] ? args[3] : '' )
+      tag_var         = tag.empty? ? '' : tag.gsub(/\_/, '::')
+      override_var    = tag_var.empty? ? nil : "#{tag_var}::#{type_name}"
+      default_var     = tag_var.empty? ? nil : "#{tag_var}::#{type_name}_defaults"
+      
+      options         = ( args[4] ? args[4] : {} )
+
+      config = CORL::Config.init_flat(options, [ :resource, :corl_resources ], {
+        :hiera_scope     => self,
+        :puppet_scope    => self,
+        :search          => 'core::default',
+        :init_fact       => 'hiera_ready',
+        :force           => true,
+        :merge           => true,
+        :resource_prefix => tag,
+        :title_prefix    => tag
+      })
+      
+      resources = CORL::Config.normalize(resources, override_var, config)
+      defaults  = CORL::Config.normalize(defaults, default_var, config)
+          
+      CORL.provisioner(:puppet).add(definition_name, resources, defaults, config)
+    end
+  end
+end

data/lib/puppet/parser/functions/deep_merge.rb

@@ -0,0 +1,21 @@
+#
+# deep_merge.rb
+#
+# Merges multiple hashes together recursively.
+#
+module Puppet::Parser::Functions
+  newfunction(:deep_merge, :type => :rvalue, :doc => <<-EOS
+This function Merges multiple hashes together recursively.
+    EOS
+) do |args|
+    
+    value = nil
+    CORL.run do
+      raise(Puppet::ParseError, "deep_merge(): Define at least one hash " +
+        "given (#{args.size} for 1)") if args.size < 1
+      
+      value = CORL::Util::Data.merge(args)
+    end
+    return value
+  end
+end

data/lib/puppet/parser/functions/ensure.rb

@@ -0,0 +1,29 @@
+#
+# ensure.rb
+#
+# Checks a given test and returns the success value or a failure value based on test results.
+#
+module Puppet::Parser::Functions
+  newfunction(:ensure, :type => :rvalue, :doc => <<-EOS
+This function checks a given test and returns the success value or a failure value based on test results.
+    EOS
+) do |args|
+    
+    value = nil
+    CORL.run do
+      raise(Puppet::ParseError, "ensure(): Must have at least a test and optional success and failure values specified; " +
+        "given (#{args.size} for 1)") if args.size < 1
+      
+      test          = args[0]
+      success_value = (args.size > 1 ? args[1] : test)
+      failure_value = (args.size > 2 ? args[2] : :undef)
+      
+      #dbg(test, 'test')
+      #dbg(success_value, 'success')
+      
+      value = CORL::Util::Data.ensure(test, success_value, failure_value)
+      #dbg(value, 'value')
+    end
+    return value
+  end
+end

data/lib/puppet/parser/functions/file_exists.rb

@@ -0,0 +1,19 @@
+#
+# file_exists.rb
+#
+module Puppet::Parser::Functions
+  newfunction(:file_exists, :type => :rvalue, :doc => <<-EOS
+Returns an boolean value if a given file and/or directory exists on Puppet Master.
+EOS
+  ) do |args|
+    
+    value = nil
+    CORL.run do
+      raise(Puppet::ParseError, "file_exists(): Must have a file or directory name specified; " +
+        "given (#{args.size} for 1)") if args.size < 1
+      
+      value = CORL::Data::Disk.exists?(args[0])
+    end
+    return value
+  end
+end

data/lib/puppet/parser/functions/global_array.rb

@@ -0,0 +1,35 @@
+#
+# global_array.rb
+#
+# See: global_param.rb
+#
+module Puppet::Parser::Functions
+  newfunction(:global_array, :type => :rvalue, :doc => <<-EOS
+This function performs a lookup for a variable value in various locations:
+See: global_params()
+If no value is found in the defined sources, it returns an empty array ([])
+    EOS
+) do |args|
+    
+    value = nil
+    CORL.run do
+      raise(Puppet::ParseError, "global_array(): Define at least the variable name " +
+        "given (#{args.size} for 1)") if args.size < 1
+    
+      var_name = args[0]
+      default  = ( args.size > 1 ? args[1] : [] ) 
+      options  = ( args.size > 2 ? args[2] : {} )
+    
+      config = CORL::Config.init_flat(options, [ :param, :global_array ], {
+        :hiera_scope  => self,
+        :puppet_scope => self,
+        :search       => 'core::default',
+        :init_fact    => 'hiera_ready',
+        :force        => true,
+        :merge        => true
+      })
+      value = CORL::Config.lookup_array(var_name, default, config)
+    end
+    return value
+  end
+end

data/lib/puppet/parser/functions/global_hash.rb

@@ -0,0 +1,35 @@
+#
+# global_hash.rb
+#
+# See: global_param.rb
+#
+module Puppet::Parser::Functions
+  newfunction(:global_hash, :type => :rvalue, :doc => <<-EOS
+This function performs a lookup for a variable value in various locations:
+See: global_params()
+If no value is found in the defined sources, it returns an empty hash ({})
+    EOS
+) do |args|
+    
+    value = nil
+    CORL.run do
+      raise(Puppet::ParseError, "global_hash(): Define at least the variable name " +
+        "given (#{args.size} for 1)") if args.size < 1
+
+      var_name = args[0]
+      default  = ( args.size > 1 ? args[1] : {} )  
+      options  = ( args.size > 2 ? args[2] : {} )
+    
+      config = CORL::Config.init_flat(options, [ :param, :global_hash ], {
+        :hiera_scope  => self,
+        :puppet_scope => self,
+        :search       => 'core::default',
+        :init_fact    => 'hiera_ready',
+        :force        => true,
+        :merge        => true
+      })
+      value = CORL::Config.lookup_hash(var_name, default, config)
+    end
+    return value
+  end
+end

data/lib/puppet/parser/functions/global_options.rb

@@ -0,0 +1,23 @@
+#
+# global_options.rb
+#
+# This function sets globally available default options for other functions.
+#
+module Puppet::Parser::Functions
+  newfunction(:global_options, :doc => <<-EOS
+This function sets globally available default options for other functions:
+EOS
+) do |args|
+    
+    CORL.run do
+      raise(Puppet::ParseError, "global_options(): Define a context name and at least one option name/value pair: " +
+        "given (#{args.size} for 2)") if args.size < 2
+
+      contexts = args[0]
+      options  = args[1]
+      force    = ( args[2] ? true : false )
+      
+      CORL::Config.set_options(contexts, options, force)
+    end
+  end
+end