corl 0.4.3 → 0.4.4

data/Gemfile.lock

@@ -10,20 +10,28 @@ GEM
     deep_merge (1.0.1)
     descendants_tracker (0.0.3)
     diff-lcs (1.2.5)
-    excon (0.31.0)
+    excon (0.32.1)
     facter (1.7.5)
     faraday (0.9.0)
       multipart-post (>= 1.2, < 3)
     ffi (1.9.3)
-    fog (1.20.0)
+    fog (1.21.0)
+      fog-brightbox
+      fog-core (~> 1.21, >= 1.21.1)
+      fog-json
+      nokogiri (~> 1.5, >= 1.5.11)
+    fog-brightbox (0.0.1)
+      fog-core
+      fog-json
+    fog-core (1.21.1)
       builder
-      excon (~> 0.31.0)
+      excon (~> 0.32)
       formatador (~> 0.2.0)
       mime-types
-      multi_json (~> 1.0)
       net-scp (~> 1.1)
       net-ssh (>= 2.1.3)
-      nokogiri (>= 1.5.11)
+    fog-json (1.0.0)
+      multi_json (~> 1.0)
     formatador (0.2.4)
     git (1.2.6)
     github_api (0.11.3)
@@ -68,7 +76,7 @@ GEM
     netrc (0.7.7)
     nokogiri (1.6.1)
       mini_portile (~> 0.5.0)
-    nucleon (0.1.6)
+    nucleon (0.1.7)
       celluloid (~> 0.15)
       childprocess (~> 0.5.0)
       deep_merge (~> 1.0)

data/README.rdoc

@@ -1,10 +1,10 @@
-= CORL (Cluster Orchestration and Research Library)
+=== CORL (Coral Orchestration and Research Library)
 
 Coming soon!
 
 Note:  This library is still very early in development!
 
-== Contributing to CORL
+==== Contributing to CORL
  
 * Check out the latest {major}.{minor} branch to make sure the feature hasn't 
   been implemented or the bug hasn't been fixed yet.
@@ -19,9 +19,9 @@ Note:  This library is still very early in development!
   to have your own version, or is otherwise necessary, that is fine, but 
   please isolate to its own commit so I can cherry-pick around it.
 
-== Copyright
+==== Copyright
 
 Licensed under GPLv3.  See LICENSE.txt for further details.
 
 Copyright (c) 2013  Adrian Webb <adrian.webb@coralnexus.com>
-Coral Technology Group LLC
+Coral Technology Group LLC

data/VERSION

@@ -1 +1 @@
-0.4.3
+0.4.4

data/corl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "corl"
-  s.version = "0.4.3"
+  s.version = "0.4.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Adrian Webb"]
-  s.date = "2014-03-16"
+  s.date = "2014-03-19"
   s.description = "Framework that provides a simple foundation for growing organically in the cloud"
   s.email = "adrian.webb@coralnexus.com"
   s.executables = ["corl"]
@@ -50,6 +50,7 @@ Gem::Specification.new do |s|
     "lib/CORL/action/exec.rb",
     "lib/CORL/action/image.rb",
     "lib/CORL/action/images.rb",
+    "lib/CORL/action/keypair.rb",
     "lib/CORL/action/lookup.rb",
     "lib/CORL/action/machines.rb",
     "lib/CORL/action/provision.rb",

data/lib/CORL/action/keypair.rb

@@ -0,0 +1,56 @@
+
+module CORL
+module Action
+class Keypair < Plugin::CloudAction
+  
+  include Mixin::Action::Keypair
+ 
+  #----------------------------------------------------------------------------
+  # Settings
+  
+  def configure
+    super do
+      codes :key_failure
+      
+      register :json, :bool, true
+      keypair_config
+    end
+  end
+  
+  #---
+  
+  def ignore
+    node_ignore
+  end
+  
+  #-----------------------------------------------------------------------------
+  # Operations
+ 
+  def execute
+    super do |node, network|
+      if keys = keypair
+        ui.info("\n", { :prefix => false })
+        ui_group(Util::Console.cyan("#{keys.type.upcase} SSH keypair")) do |ui|
+          ui.info("-----------------------------------------------------")
+        
+          if settings[:json]
+            private_key = Util::Console.blue(Util::Data.to_json(keys.encrypted_key, true))
+            ssh_key     = keys.ssh_key.gsub(/^ssh\-[a-z]+\s+/, '')           
+            ssh_key     = Util::Console.green(Util::Data.to_json(ssh_key, true))
+          else
+            private_key = Util::Console.blue(keys.encrypted_key)
+            ssh_key     = Util::Console.green(keys.ssh_key)       
+          end
+          
+          ui.info("SSH private key:\n#{private_key}")
+          ui.info("SSH public key:\n#{ssh_key}")
+          ui.info("\n", { :prefix => false }) 
+        end          
+      else
+        myself.status = code.key_failure  
+      end
+    end
+  end
+end
+end
+end

data/lib/CORL/action/provision.rb

@@ -10,6 +10,9 @@ class Provision < Plugin::CloudAction
     super do
       codes :network_failure,
             :provision_failure
+            
+      register :dry_run, :bool, false
+      register :color, :bool, true
     end
   end
   
@@ -36,10 +39,8 @@ class Provision < Plugin::CloudAction
               profiles      = provider_info[:profiles]
           
               collection.each do |name, provisioner|
-                build_profiles = provisioner.build_profiles
-            
                 if supported_profiles = provisioner.supported_profiles(profiles)
-                  profile_success = provisioner.provision(supported_profiles)
+                  profile_success = provisioner.provision(supported_profiles, settings)
                   success         = false unless profile_success
                 end
               end

data/lib/CORL/provisioner/puppetnode.rb

@@ -1,4 +1,4 @@
-
+    
 module CORL
 module Provisioner
 class Puppetnode < CORL.plugin_class(:provisioner)
@@ -12,10 +12,33 @@ class Puppetnode < CORL.plugin_class(:provisioner)
     super do
       if CORL.log_level == :debug
         Puppet.debug = true
-      end   
-    end
+      end
+      unless reload
+        Puppet::Util::Log.newdesttype id do
+          def handle(msg)
+            levels = {
+              :emerg   => { :name => 'emergency', :send => :error },
+              :alert   => { :name => 'alert', :send => :error },
+              :crit    => { :name => 'critical', :send => :error },
+              :err     => { :name => 'error', :send => :error },
+              :warning => { :name => 'warning', :send => :warn },
+              :notice  => { :name => 'notice', :send => :success },
+              :info    => { :name => 'info', :send => :info },
+              :debug   => { :name => 'debug', :send => :info }
+            }
+            str   = msg.respond_to?(:multiline) ? msg.multiline : msg.to_s
+            str   = msg.source == "Puppet" ? str : "#{msg.source}: #{str}"
+            level = levels[msg.level]
+        
+            CORL.ui_group("puppetnode::#{name}(#{level[:name]})") do |ui|        
+              ui.send(level[:send], str)
+            end
+          end
+        end
+      end
+    end  
   end
-  
+    
   #---
   
   def register(options = {})
@@ -42,12 +65,13 @@ class Puppetnode < CORL.plugin_class(:provisioner)
   def init_puppet(profiles)
     locations = build_locations
     
-    # Must be in this order!!
     Puppet.initialize_settings
-    Puppet::Util::Log.newdestination(:console)
+    Puppet::Util::Log.newdestination(id)
     
     # TODO: Figure out how to store these damn settings in a specialized
     # environment without phantom empty environment issues.
+    
+    Puppet[:default_file_terminus] = :file_server
       
     node = get_node    
     Puppet[:node_name_value] = id.to_s
@@ -70,10 +94,7 @@ class Puppetnode < CORL.plugin_class(:provisioner)
     
     @compiler = Puppet::Parser::Compiler.new(node)
     register
-     
-    # Initialize the compiler so we can can lookup and include stuff
-    # This is ugly but seems to be the only way.
-    compiler.compile
+    node
   end
   protected :init_puppet
   
@@ -158,15 +179,6 @@ class Puppetnode < CORL.plugin_class(:provisioner)
   
   #---
   
-  def include(resource_name, properties = {}, options = {})
-    Util::Puppet.include(resource_name, properties, Config.ensure(options).defaults({
-      :provisioner  => :puppetnode, 
-      :puppet_scope => scope      
-    }))
-  end
-  
-  #---
-  
   def add_search_path(type, resource_name)
     Config.set_options([ :all, type ], { :search => [ resource_name.to_s ] })  
   end
@@ -174,81 +186,96 @@ class Puppetnode < CORL.plugin_class(:provisioner)
   #---
     
   def provision(profiles, options = {})
-    locations = build_locations
-    success   = true
+    super do |config|
+      locations = build_locations
+      success   = true
     
-    include_location = lambda do |type, add_to_catalog = true, add_search_path = false|      
-      locations[:package].each do |name, package_directory|
-        gateway       = File.join(build_directory, package_directory, "#{type}.pp")
-        resource_name = concatenate([ name, type ])
+      include_location = lambda do |type, parameters = {}, add_search_path = false|
+        classes = {}
+            
+        locations[:package].each do |name, package_directory|
+          gateway       = File.join(build_directory, package_directory, "#{type}.pp")
+          resource_name = concatenate([ name, type ])
         
-        add_search_path(type, resource_name) if add_search_path
+          add_search_path(type, resource_name) if add_search_path
         
-        if File.exists?(gateway)
-          import(gateway)
-          include(resource_name, { :before => 'Anchor[gateway_init]' }) if add_to_catalog                   
-        end
+          if File.exists?(gateway)
+            import(gateway)
+            classes[resource_name] = parameters                 
+          end
         
-        directory = File.join(build_directory, package_directory, type.to_s)
-        Dir.glob(File.join(directory, '*.pp')).each do |file|
-          resource_name = concatenate([ name, type, File.basename(file).gsub('.pp', '') ])
-          import(file)
-          include(resource_name, { :before => 'Anchor[gateway_init]' }) if add_to_catalog
-        end        
-      end
+          directory = File.join(build_directory, package_directory, type.to_s)
+          Dir.glob(File.join(directory, '*.pp')).each do |file|
+            resource_name = concatenate([ name, type, File.basename(file).gsub('.pp', '') ])
+            import(file)
+            classes[resource_name] = parameters
+          end        
+        end
     
-      gateway       = File.join(build_directory, locations[:build], "#{type}.pp")
-      resource_name = concatenate([ plugin_name, type ])
+        gateway       = File.join(build_directory, locations[:build], "#{type}.pp")
+        resource_name = concatenate([ plugin_name, type ])
       
-      add_search_path(type, resource_name) if add_search_path
+        add_search_path(type, resource_name) if add_search_path
      
-      if File.exists?(gateway)
-        import(gateway)
-        include(resource_name, { :before => 'Anchor[gateway_init]' }) if add_to_catalog                   
-      end
+        if File.exists?(gateway)
+          import(gateway)
+          classes[resource_name] = parameters                
+        end
     
-      if locations.has_key?(type)
-        directory = File.join(build_directory, locations[type])
-        Dir.glob(File.join(directory, '*.pp')).each do |file|
-          resource_name = concatenate([ plugin_name, type, File.basename(file).gsub('.pp', '') ])
-          import(file)
-          include(resource_name, { :before => 'Anchor[gateway_init]' }) if add_to_catalog
+        if locations.has_key?(type)
+          directory = File.join(build_directory, locations[type])
+          Dir.glob(File.join(directory, '*.pp')).each do |file|
+            resource_name = concatenate([ plugin_name, type, File.basename(file).gsub('.pp', '') ])
+            import(file)
+            classes[resource_name] = parameters
+          end
         end
+        classes
       end
-    end
     
-    @@puppet_lock.synchronize do
-      begin
-        start_time = Time.now
-        
-        init_puppet(profiles)
+      @@puppet_lock.synchronize do
+        begin          
+          ui.info("Starting catalog generation")
+          
+          start_time = Time.now        
+          node       = init_puppet(profiles)
         
-        # Include defaults
-        include_location.call(:default, true, true)
+          # Include defaults
+          classes = include_location.call(:default, {}, true)
       
-        # Import and include needed profiles
-        include_location.call(:profiles, false)
+          # Import needed profiles
+          include_location.call(:profiles, {}, false)
       
-        profiles.each do |profile|
-          include(profile, { :require => 'Anchor[gateway_exit]' })
-        end
+          profiles.each do |profile|
+            classes[profile.to_s] = { :require => 'Anchor[profile_start]' }
+          end
+        
+          # Compile catalog
+          node.classes = classes
+          compiler.compile
         
-        # Start system configuration 
-        catalog = compiler.catalog.to_ral
-        catalog.finalize
-        catalog.retrieval_duration = Time.now - start_time
+          # Start system configuration
+          catalog = compiler.catalog.to_ral
+          catalog.finalize
+          catalog.retrieval_duration = Time.now - start_time
+          
+          unless config.get(:dry_run, false)
+            ui.info("\n", { :prefix => false })
+            ui.info("Starting configuration run")
+                        
+            configurer = Puppet::Configurer.new
+            if ! configurer.run(:catalog => catalog, :pluginsync => false)
+              success = false
+            end
+          end
         
-        configurer = Puppet::Configurer.new
-        if ! configurer.run(:catalog => catalog, :pluginsync => false)
-          success = false
+        rescue Exception => error
+          raise error
+          Puppet.log_exception(error)
         end
-      
-      rescue Exception => error
-        raise error
-        Puppet.log_exception(error)
-      end
-    end    
-    success
+      end    
+      success
+    end
   end
   
   #-----------------------------------------------------------------------------

data/lib/core/mixin/action/keypair.rb

@@ -72,7 +72,7 @@ module Keypair
       :bits    => settings[:key_bits],
       :comment => settings[:key_comment]
     })
-    @keypair = Util::SSH.generate(options)
+    @keypair = Util::SSH.generate(config)
     settings.import({ :keypair => @keypair })
   end
   

data/lib/core/mixin/lookup.rb

@@ -32,6 +32,8 @@ module Lookup
   #---
   
   def hiera_config
+    Kernel.load File.join(File.dirname(__FILE__), '..', 'mod', 'hiera_backend.rb')
+    
     config_file = CORL.value(:hiera_config_file, File.join(Hiera::Util.config_dir, 'hiera.yaml'))
     config      = {}
 
@@ -72,7 +74,8 @@ module Lookup
     
     hiera_scope = config.get(:hiera_scope, {})
     override    = config.get(:override, nil)
-    context     = config.get(:context, :priority)    
+    context     = config.get(:context, :priority)
+    debug       = config.get(:debug, false)    
     
     return_property = config.get(:return_property, false)
     
@@ -85,8 +88,18 @@ module Lookup
       property       = property.to_sym
       first_property = property unless first_property
       
+      if debug
+        CORL.ui.info("\n", { :prefix => false })
+        CORL.ui_group(Util::Console.purple(property)) do |ui|
+          ui.info("-----------------------------------------------------")
+        end
+      end
+      
       # Try to load facts first (these can not be overridden)
-      unless value = fact(property)
+      value = fact(property)
+      debug_lookup(config, property, value, "Fact lookup")
+      
+      unless value
         if CORL.admin? 
           if config_initialized?
             # Try to find in Hiera data store (these might be security sensitive)
@@ -94,23 +107,33 @@ module Lookup
               hiera_scope = Hiera::Scope.new(hiera_scope)
             end
             value = hiera.lookup(property.to_s, nil, hiera_scope, override, context)
+            debug_lookup(config, property, value, "Hiera lookup")
           end 
     
           if provisioner && Util::Data.undef?(value)
             # Search the provisioner scope (only admins can provision a machine)
             value = CORL.provisioner({ :name => :lookup }, provisioner).lookup(property, default, config)
+            debug_lookup(config, property, value, "Provisioner lookup")
           end
         end
       end
     end
-    value = default if Util::Data.undef?(value) # Resort to default
-    value = Util::Data.value(value)
+    if Util::Data.undef?(value) # Resort to default
+      value = default
+      debug_lookup(config, first_property, value, "Default value")
+    end
+    value = Util::Data.value(value, config.get(:undefined_value, :undefined))
     
     if ! Config.get_property(first_property) || ! Util::Data.undef?(value)
       Config.set_property(first_property.to_s, value)
     end
     
-    return value, first_property if return_property
+    debug_lookup(config, first_property, value, 'Internalized value')
+    
+    if return_property
+      return value, first_property
+    end
+    CORL.ui.info("\n", { :prefix => false }) if debug
     value
   end
     
@@ -122,18 +145,24 @@ module Lookup
      
     if Util::Data.undef?(value)
       value = default
+      debug_lookup(config, property, value, "Array default value")
         
     elsif ! Util::Data.empty?(default)
       if config.get(:merge, false)
         value = Util::Data.merge([default, value], config)
+        debug_lookup(config, property, value, "Merged array value with default")
       end
     end
     
     unless value.is_a?(Array)
       value = ( Util::Data.empty?(value) ? [] : [ value ] )
     end
+    
+    value = Util::Data.value(value, config.get(:undefined_value, :undefined))
+    debug_lookup(config, property, value, "Final array value")
        
     Config.set_property(property.to_s, value)
+    CORL.ui.info("\n", { :prefix => false }) if config.get(:debug, false)
     value
   end
     
@@ -145,10 +174,12 @@ module Lookup
     
     if Util::Data.undef?(value)
       value = default
+      debug_lookup(config, property, value, "Hash default value")
         
     elsif ! Util::Data.empty?(default)
       if config.get(:merge, false)
         value = Util::Data.merge([default, value], config)
+        debug_lookup(config, property, value, "Merged hash value with default")
       end
     end
     
@@ -156,7 +187,11 @@ module Lookup
       value = ( Util::Data.empty?(value) ? {} : { :value => value } )
     end
     
+    value = Util::Data.value(value, config.get(:undefined_value, :undefined))
+    debug_lookup(config, property, value, "Final hash value")
+    
     Config.set_property(property.to_s, value)
+    CORL.ui.info("\n", { :prefix => false }) if config.get(:debug, false)
     value
   end
   
@@ -196,6 +231,22 @@ module Lookup
     end    
     results
   end
+  
+  #---
+  
+  def debug_lookup(config, property, value, label)
+    if config.get(:debug, false)
+      CORL.ui_group(Util::Console.cyan(property.to_s)) do |ui|
+        dump = Util::Console.green(Util::Data.to_json(value, true))
+        
+        if dump.match(/\n+/)
+          ui.info("#{label}:\n#{dump}")  
+        else
+          ui.info("#{label}: #{dump}")  
+        end        
+      end
+    end
+  end
 end
 end
 end