corl 0.4.29 → 0.5.0

data/lib/CORL/builder/package.rb

@@ -0,0 +1,75 @@
+
+module CORL
+module Builder
+class Package < CORL.plugin_class(:CORL, :builder)
+  
+  #-----------------------------------------------------------------------------
+  # Package plugin interface
+ 
+  def normalize(reload)
+    super do
+      @packages = {}
+    end
+  end
+
+  #-----------------------------------------------------------------------------
+  # Property accessors / modifiers
+  
+  def build_directory
+    File.join(network.build_directory, 'packages')
+  end
+  
+  #---
+  
+  def packages
+    @packages
+  end
+  
+  def set_package(name, directory)
+    @packages[name] = directory
+  end
+ 
+  #-----------------------------------------------------------------------------
+  # Package interface operations
+  
+  def build_provider(name, project_reference, environment)
+    provider_id = id(name)
+    directory   = File.join(internal_path(build_directory), provider_id.to_s)
+    success     = true
+      
+    ui.info("Building package #{blue(name)} at #{purple(project_reference)} into #{green(directory)}")
+      
+    full_directory = File.join(network.directory, directory)
+      
+    unless packages.has_key?(provider_id)
+      project = build_config.manage(:configuration, extended_config(:package, {
+        :directory     => full_directory,
+        :url           => project_reference,
+        :create        => File.directory?(full_directory) ? false : true,
+        :manage_ignore => false
+      }))
+      unless project
+        ui.warn("Package #{cyan(name)} failed to initialize")
+        success = false
+      end
+           
+      if success
+        set_package(provider_id, full_directory)
+        
+        build_config.import(project)
+        build_config.set_location(plugin_provider, name, directory)
+      
+        if project.get([ :builders, plugin_provider ], false)
+          sub_packages = process_environment(project.get_hash([ :builders, plugin_provider ]), environment)          
+          
+          status  = parallel(:build_provider, sub_packages, environment)          
+          success = false if status.values.include?(false)
+        end
+      end
+    end
+    ui.success("Build of package #{blue(name)} finished") if success
+    success
+  end
+end
+end
+end

data/lib/CORL/configuration/file.rb

@@ -1,7 +1,7 @@
 
 module CORL
 module Configuration
-class File < CORL.plugin_class(:configuration)
+class File < CORL.plugin_class(:CORL, :configuration)
 
   #-----------------------------------------------------------------------------
   # Configuration plugin interface
@@ -84,7 +84,7 @@ class File < CORL.plugin_class(:configuration)
   
   def separate    
     file_data        = Config.new
-    default_provider = CORL.type_default(:translator)
+    default_provider = CORL.type_default(:nucleon, :translator)
     
     split_config = lambda do |properties, local_router, parents = []|
       properties.each do |name, value|
@@ -234,7 +234,7 @@ class File < CORL.plugin_class(:configuration)
       begin
         FileUtils.mkdir_p(attach_path) unless Dir.exists?(attach_path)
       
-      rescue Exception => error
+      rescue => error
         alert(error.message)
         success = false
       end
@@ -258,7 +258,7 @@ class File < CORL.plugin_class(:configuration)
             FileUtils.mkdir_p(attach_path) unless Dir.exists?(attach_path)
             FileUtils.cp(file, new_file)
           
-          rescue Exception => error
+          rescue => error
             alert(error.message)
             success = false
           end  
@@ -318,7 +318,7 @@ class File < CORL.plugin_class(:configuration)
       logger.debug("Clearing configuration file information")      
       search.clear
     else
-      translators = CORL.loaded_plugins(:translator)
+      translators = CORL.loaded_plugins(:nucleon, :translator)
       file_bases  = [ "corl", extension_collect(:base) ].flatten
       
       project.localize do

data/lib/CORL/machine/{aws.rb → AWS.rb}

@@ -1,7 +1,7 @@
 
 module CORL
 module Machine
-class Aws < Fog
+class AWS < Fog
  
   #-----------------------------------------------------------------------------
   # Checks
@@ -150,4 +150,4 @@ class Aws < Fog
   end
 end
 end
-end
+end

data/lib/CORL/machine/physical.rb

@@ -1,7 +1,7 @@
 
 module CORL
 module Machine
-class Physical < CORL.plugin_class(:machine)
+class Physical < CORL.plugin_class(:CORL, :machine)
   
   #-----------------------------------------------------------------------------
   # Machine plugin interface

data/lib/CORL/machine/rackspace.rb

@@ -60,4 +60,4 @@ class Rackspace < Fog
   end
 end
 end
-end
+end

data/lib/CORL/machine/vagrant.rb

@@ -1,7 +1,7 @@
 
 module CORL
 module Machine
-class Vagrant < CORL.plugin_class(:machine)
+class Vagrant < CORL.plugin_class(:CORL, :machine)
   
   include Mixin::Machine::SSH
   
@@ -140,7 +140,7 @@ class Vagrant < CORL.plugin_class(:machine)
   def reload(options = {})
     super do |config|
       success = run(:reload, config)
-      success = init_ssh_session(true, config.get(:tries, 5), config.get(:sleep_time, 5)) if success
+      success = init_ssh_session(true, config.get(:tries, 12), config.get(:sleep_time, 5)) if success
     end
   end
 

data/lib/CORL/network/{default.rb → CORL.rb}

@@ -1,7 +1,7 @@
 
 module CORL
 module Network
-class Default < CORL.plugin_class(:network)
+class CORL < CORL.plugin_class(:CORL, :network)
  
   #-----------------------------------------------------------------------------
   # Cloud plugin interface

data/lib/CORL/node/{aws.rb → AWS.rb}

@@ -1,7 +1,7 @@
 
 module CORL
 module Node
-class Aws < Fog
+class AWS < Fog
  
   #-----------------------------------------------------------------------------
   # Node plugin interface

data/lib/CORL/node/local.rb

@@ -1,7 +1,7 @@
 
 module CORL
 module Node
-class Local < CORL.plugin_class(:node)
+class Local < CORL.plugin_class(:CORL, :node)
  
   #-----------------------------------------------------------------------------
   # Node plugin interface

data/lib/CORL/node/vagrant.rb

@@ -1,7 +1,7 @@
 
 module CORL
 module Node
-class Vagrant < CORL.plugin_class(:node)
+class Vagrant < CORL.plugin_class(:CORL, :node)
  
   #-----------------------------------------------------------------------------
   # Node plugin interface
@@ -227,10 +227,15 @@ class Vagrant < CORL.plugin_class(:node)
       if data.include?('stdin: is not a tty') || data.include?('unable to re-open stdin')
         data = ''  
       end
+    elsif type == :output
+      # Hide redundant Facter output
+      if data =~ /^Already evaluated [a-z]+ at [^,]+, reevaluating anyways$/
+        data = ''
+      end  
     end
     data  
   end
-   
+  
   #-----------------------------------------------------------------------------
   # Machine type utilities
   

data/lib/CORL/provisioner/puppetnode.rb

@@ -1,6 +1,7 @@
+
 module CORL
 module Provisioner
-class Puppetnode < CORL.plugin_class(:provisioner)
+class Puppetnode < CORL.plugin_class(:CORL, :provisioner)
   
   @@puppet_lock = Mutex.new
   
@@ -12,6 +13,16 @@ class Puppetnode < CORL.plugin_class(:provisioner)
     @@status
   end
   
+  #---
+  
+  def self.network
+    @@network
+  end
+  
+  def self.node
+    @@node
+  end
+  
   #-----------------------------------------------------------------------------
   # Provisioner plugin interface
    
@@ -72,124 +83,107 @@ class Puppetnode < CORL.plugin_class(:provisioner)
   
   #-----------------------------------------------------------------------------
   # Puppet initialization
+  
+  def init_puppet(node, profiles)    
+    Puppet.initialize_settings
     
-  def init_puppet(profiles)
-    locations = build_locations
+    apply_environment = nil
+    locations         = build_locations(node)
     
-    Puppet.initialize_settings
-    Puppet::Util::Log.newdestination(id)
+    environment, environment_directory = ensure_environment(node)    
+    
+    Puppet::Util::Log.newdestination(id)    
+    Puppet::Transaction::Report.indirection.cache_class = :yaml
     
-    # TODO: Figure out how to store these damn settings in a specialized
-    # environment without phantom empty environment issues.
+    Puppet[:graph] = true if CORL.log_level == :error
     
-    Puppet[:data_binding_terminus] = 'corl'
+    Puppet[:node_terminus]         = :plain
+    Puppet[:data_binding_terminus] = :corl
     Puppet[:default_file_terminus] = :file_server
-    Puppet[:node_name_value]       = id.to_s
     
     unless profiles.empty?
       modulepath = profiles.collect do |profile|
-        File.join(build_directory, locations[:module][profile.to_sym])
-      end
-      Puppet[:modulepath] = array(modulepath).join(File::PATH_SEPARATOR)
+        profile_directory = File.join(network.directory, locations[:puppet_module][profile.to_sym])
+        File.directory?(profile_directory) ? profile_directory : nil
+      end.compact
     end
     
     if manifest = gateway
       if manifest.match(/^packages\/.*/)
-        manifest = File.join(build_directory, locations[:build], manifest)
+        manifest = File.join(network.build_directory, manifest)
       else
         manifest = File.join(network.directory, directory, manifest)
       end
-      Puppet[:manifest] = manifest
     end
     
-    node      = get_node
-    @compiler = Puppet::Parser::Compiler.new(node)
-    register
-    node
+    Puppet[:environment] = environment
+    Puppet::Node::Environment.create(environment.to_sym, modulepath, manifest)
   end
   protected :init_puppet
-  
+    
   #---
    
-  def get_node
-    node_id = id.to_s
-    node = Puppet::Node.indirection.find(node_id)
-         
-    if facts = Puppet::Node::Facts.indirection.find(node_id)
-      facts.name = node_id
-      node.merge(facts.values)
-    end
-    node
+  def get_puppet_node(environment)
+    Puppet[:node_name_value] = id.to_s
+    
+    puppet_node = Puppet::Node.indirection.find(id.to_s, :environment => environment)
+    
+    puppet_node.merge(string_map(@@node.facts))    
+    puppet_node
   end
-  protected :get_node
+  protected :get_puppet_node
   
   #-----------------------------------------------------------------------------
   # Provisioner interface operations
   
-  def build(options = {})
-    super do |locations, package_info|
-      success = true
-      
-      # Build modules
-      locations[:module] = {}
-      
-      init_profile = lambda do |package_name, profile_name, profile_info|
-        package_id = id(package_name)
-        base_directory = File.join(locations[:build], 'modules', package_id.to_s, profile_name.to_s)
-        profile_success = true
+  def build_profile(name, info, package, environment, profiles)
+    super do |processed_info|
+      package_id = id(package)
+      directory  = File.join(internal_path(build_directory), package_id.to_s, name.to_s)
+      success    = true
         
-        ui.info("Building CORL profile #{blue(profile_name)} modules into #{green(base_directory)}")
+      ui.info("Building CORL profile #{blue(name)} modules into #{green(directory)}")
+    
+      if processed_info.has_key?(:modules)
+        status  = parallel(:build_module, hash(processed_info[:modules]), directory, name, environment)
+        success = status.values.include?(false) ? false : true
         
-        if profile_info.has_key?(:modules)
-          profile_info[:modules].each do |module_name, module_reference|
-            module_directory = File.join(base_directory, module_name.to_s)
-            
-            ui.info("Building Puppet module #{blue(module_name)} at #{purple(module_reference)} into #{green(module_directory)}")
-            
-            full_module_directory = File.join(build_directory, module_directory)
-                
-            module_project = CORL.project(extended_config(:puppet_module, {
-              :directory => full_module_directory,
-              :url => module_reference,
-              :create => File.directory?(full_module_directory) ? false : true,
-              :pull => true
-            }))
-            unless module_project
-              ui.warn("Puppet module #{cyan(module_name)} failed to initialize")
-              profile_success = false
-              break
-            end
-          end
-          locations[:module][profile_id(package_name, profile_name)] = base_directory if profile_success
-          profile_success
-        end
-      end
-      
-      hash(package_info.get([ :provisioners, plugin_provider ])).each do |package_name, info|
-        if info.has_key?(:profiles)
-          info[:profiles].each do |profile_name, profile_info|
-            unless init_profile.call(package_name, profile_name, profile_info)
-              success = false
-              break
-            end
-          end
-        end
-      end
-      profiles.each do |profile_name, profile_info|
-        unless init_profile.call(plugin_name, profile_name, profile_info)
-          success = false
-          break
-        end
+        build_config.set_location(:puppet_module, profile_id(package, name), directory) if success
       end
+      ui.success("Build of profile #{blue(name)} finished") if success
       success
     end
   end
   
+  def build_module(name, project_reference, directory, profile, environment)
+    module_directory      = File.join(directory, name.to_s)
+    full_module_directory = File.join(network.directory, module_directory)
+    module_project        = nil
+    success               = true
+            
+    ui.info("Building #{blue(profile)} Puppet module #{blue(name)} at #{purple(project_reference)} into #{green(module_directory)}")
+    
+    module_project = build_config.manage(:project, extended_config(:puppet_module, {
+      :directory     => full_module_directory,
+      :url           => project_reference,
+      :create        => File.directory?(full_module_directory) ? false : true,
+      :pull          => true,
+      :internal_ip   => CORL.public_ip, # Needed for seeding Vagrant VMs
+      :manage_ignore => false
+    }))
+    unless module_project
+      ui.warn("Puppet module #{cyan(name)} failed to initialize")
+      success = false
+    end
+    ui.success("Build of #{blue(profile)} #{blue(name)} finished") if success
+    success
+  end
+  
   #---
   
   def lookup(property, default = nil, options = {})
     Util::Puppet.lookup(property, default, Config.ensure(options).defaults({
-      :provisioner => :puppetnode,
+      :provisioner  => :puppetnode,
       :puppet_scope => scope
     }))
   end
@@ -198,7 +192,7 @@ class Puppetnode < CORL.plugin_class(:provisioner)
   
   def import(files, options = {})
     Util::Puppet.import(files, Config.ensure(options).defaults({
-      :puppet_scope => scope,
+      :puppet_scope       => scope,
       :puppet_import_base => network.directory
     }))
   end
@@ -211,18 +205,18 @@ class Puppetnode < CORL.plugin_class(:provisioner)
    
   #---
     
-  def provision(profiles, options = {})
-    super do |config|
-      locations = build_locations
-      success = true
+  def provision(node, profiles, options = {})
+    super do |processed_profiles, config|
+      locations = build_locations(node)
+      success   = true
     
       include_location = lambda do |type, parameters = {}, add_search_path = false|
         classes = {}
             
         locations[:package].each do |name, package_directory|
-          type_gateway = File.join(build_directory, package_directory, "#{type}.pp")
-          resource_name = concatenate([ name, type ])
-        
+          type_gateway = File.join(network.directory, package_directory, "#{type}.pp")
+          resource_name = resource([ name, type ])
+          
           add_search_path(type, resource_name) if add_search_path
         
           if File.exists?(type_gateway)
@@ -230,16 +224,16 @@ class Puppetnode < CORL.plugin_class(:provisioner)
             classes[resource_name] = parameters
           end
         
-          type_directory = File.join(build_directory, package_directory, type.to_s)
+          type_directory = File.join(network.directory, package_directory, type.to_s)
           Dir.glob(File.join(type_directory, '*.pp')).each do |file|
-            resource_name = concatenate([ name, type, File.basename(file).gsub('.pp', '') ])
+            resource_name = resource([ name, type, File.basename(file).gsub('.pp', '') ])
             import(file)
             classes[resource_name] = parameters
           end
         end
     
         type_gateway = File.join(directory, "#{type}.pp")
-        resource_name = concatenate([ plugin_name, type ])
+        resource_name = resource([ plugin_name, type ])
       
         add_search_path(type, resource_name) if add_search_path
      
@@ -252,57 +246,69 @@ class Puppetnode < CORL.plugin_class(:provisioner)
         
         if File.directory?(type_directory)
           Dir.glob(File.join(type_directory, '*.pp')).each do |file|
-            resource_name = concatenate([ plugin_name, type, File.basename(file).gsub('.pp', '') ])
+            resource_name = resource([ plugin_name, type, File.basename(file).gsub('.pp', '') ])
             import(file)
             classes[resource_name] = parameters
           end
         end
         classes
       end
-    
+      
       @@puppet_lock.synchronize do
         begin
           ui.info("Starting catalog generation")
           
           @@status[id] = code.success
+          @@network    = network    
+          @@node       = node
           
-          start_time = Time.now
-          node = init_puppet(profiles)
-        
-          # Include defaults
-          classes = include_location.call(:default, {}, true)
+          start_time        = Time.now
+          apply_environment = init_puppet(node, processed_profiles)
+          
+          Puppet.override(:environments => Puppet::Environments::Static.new(apply_environment)) do
+            puppet_node = get_puppet_node(apply_environment.name)                 
+            @compiler   = Puppet::Parser::Compiler.new(puppet_node)
+            
+            # Register Puppet module plugins
+            register
+            
+            # Include defaults
+            classes = include_location.call(:default, {}, true)
       
-          # Import needed profiles
-          include_location.call(:profiles, {}, false)
+            # Import needed profiles
+            include_location.call(:profiles, {}, false)
       
-          profiles.each do |profile|
-            classes[profile.to_s] = { :require => 'Anchor[profile_start]' }
-          end
-          
-          # Compile catalog
-          node.classes = classes
-          compiler.compile
-        
-          # Start system configuration
-          catalog = compiler.catalog.to_ral
-          catalog.finalize
-          catalog.retrieval_duration = Time.now - start_time
-          
-          unless config.get(:dry_run, false)
-            ui.info("\n", { :prefix => false })
-            ui.info("Starting configuration run")
-                        
-            configurer = Puppet::Configurer.new
-            if ! configurer.run(:catalog => catalog, :pluginsync => false)
-              success = false
+            processed_profiles.each do |profile|
+              classes[profile.to_s] = { :require => 'Anchor[profile_start]' }
+            end
+            
+            puppet_node.classes = classes
+            
+            # Compile catalog
+            compiler.compile
+            
+            catalog = compiler.catalog.to_ral
+            catalog.finalize
+            catalog.retrieval_duration = Time.now - start_time
+            
+            unless config.get(:dry_run, false)
+              ui.info("\n", { :prefix => false })
+              ui.info("Starting configuration run")
+              
+              # Configure the machine
+              configurer = Puppet::Configurer.new
+              if ! configurer.run(:catalog => catalog, :pluginsync => false)
+                success = false
+              end
             end
           end
         
         rescue Exception => error
-          raise error
           Puppet.log_exception(error)
+          raise error
         end
       end
+      
       success = false if @@status[id] != code.success
       success
     end
@@ -311,15 +317,17 @@ class Puppetnode < CORL.plugin_class(:provisioner)
   #-----------------------------------------------------------------------------
   # Utilities
   
-  def profile_id(package_name, profile_name)
-    concatenate([ package_name, 'profile', profile_name ], false)
-  end
-   
-  #---
-  
-  def concatenate(components, capitalize = false)
-    super(components, capitalize, '::')
+  def ensure_environment(node)
+    base_directory = Puppet[:environmentpath]
+    environment    = node.lookup(:corl_environment)
+    env_directory  = File.join(base_directory, environment)
+    
+    FileUtils.mkdir_p(env_directory)
+    FileUtils.mkdir_p(File.join(env_directory, 'manifests'))
+    FileUtils.mkdir_p(File.join(env_directory, 'modules'))
+    [ environment, env_directory ]
   end
+  protected :ensure_environment
 end
 end
 end