corl 0.4.4 → 0.4.5

data/Gemfile.lock

@@ -67,7 +67,7 @@ GEM
     log4r (1.1.10)
     mime-types (1.25.1)
     mini_portile (0.5.2)
-    multi_json (1.9.0)
+    multi_json (1.9.2)
     multi_xml (0.5.5)
     multipart-post (2.0.0)
     net-scp (1.1.2)
@@ -76,7 +76,7 @@ GEM
     netrc (0.7.7)
     nokogiri (1.6.1)
       mini_portile (~> 0.5.0)
-    nucleon (0.1.7)
+    nucleon (0.1.8)
       celluloid (~> 0.15)
       childprocess (~> 0.5.0)
       deep_merge (~> 1.0)

data/VERSION

@@ -1 +1 @@
-0.4.4
+0.4.5

data/bootstrap/os/ubuntu/06_puppet.sh

@@ -14,10 +14,9 @@ chmod 0644 /etc/apt/sources.list.d/puppet.list || exit 62
 # Install Puppet
 apt-get -y update || exit 63
 apt-get -y install puppet || exit 64
-gem install libshadow || exit 65
 
 # Set up Hiera configuration
-mkdir -p /var/corl/config || exit 66
+mkdir -p /var/corl/config || exit 65
 
 ( cat <<'EOP'
 ---
@@ -32,7 +31,7 @@ mkdir -p /var/corl/config || exit 66
 :hierarchy:
   - common
 EOP
-) > /etc/hiera.yaml || exit 67
-chmod 0440 /etc/hiera.yaml || exit 68
+) > /etc/hiera.yaml || exit 66
+chmod 0440 /etc/hiera.yaml || exit 67
 
-ln -fs /etc/hiera.yaml /etc/puppet/hiera.yaml || exit 69
+ln -fs /etc/hiera.yaml /etc/puppet/hiera.yaml || exit 68

data/corl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "corl"
-  s.version = "0.4.4"
+  s.version = "0.4.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Adrian Webb"]
-  s.date = "2014-03-19"
+  s.date = "2014-03-20"
   s.description = "Framework that provides a simple foundation for growing organically in the cloud"
   s.email = "adrian.webb@coralnexus.com"
   s.executables = ["corl"]

data/lib/CORL/action/provision.rb

@@ -12,7 +12,6 @@ class Provision < Plugin::CloudAction
             :provision_failure
             
       register :dry_run, :bool, false
-      register :color, :bool, true
     end
   end
   

data/lib/CORL/configuration/file.rb

@@ -72,10 +72,10 @@ class File < CORL.plugin_class(:configuration)
         
           if parser && raw && ! raw.empty?
             logger.debug("Source configuration file contents: #{raw}")            
-            file_properties = parser.parse(raw)
+            parse_properties = parser.parse(raw)
             
-            generate_routes.call(config_name, file_properties)
-            properties.import(file_properties)
+            generate_routes.call(config_name, parse_properties)
+            properties.import(parse_properties)
           end         
         end
       end
@@ -121,8 +121,9 @@ class File < CORL.plugin_class(:configuration)
         else
           if local_router.is_a?(String)
             # Router is a config_name string
-            file_data.set([ local_router, keys ].flatten, value)          
-          elsif router.is_a?(Hash)
+            file_data.set([ local_router, keys ].flatten, value)
+                     
+          elsif local_router.is_a?(Hash)
             # Router is a hash with sub options we have to pick from
             config_name = select_largest(local_router)
             file_data.set([ config_name, keys ].flatten, value)  
@@ -182,7 +183,7 @@ class File < CORL.plugin_class(:configuration)
       end
       if success && ! config_files.empty?
         commit_files = [ config_files, method_config.get_array(:files) ].flatten
-          
+        
         logger.debug("Source configuration rendering: #{rendering}")        
         success = update_project(commit_files, method_config)
       end

data/lib/CORL/machine/physical.rb

@@ -34,7 +34,7 @@ class Physical < CORL.plugin_class(:machine)
   #---
   
   def hostname
-    fact(:hostname)
+    fact(:fqdn)
   end
   
   #---

data/lib/CORL/provisioner/puppetnode.rb

@@ -27,10 +27,10 @@ class Puppetnode < CORL.plugin_class(:provisioner)
               :debug   => { :name => 'debug', :send => :info }
             }
             str   = msg.respond_to?(:multiline) ? msg.multiline : msg.to_s
-            str   = msg.source == "Puppet" ? str : "#{msg.source}: #{str}"
+            str   = msg.source == "Puppet" ? str : "#{CORL.blue(msg.source)}: #{str}"
             level = levels[msg.level]
         
-            CORL.ui_group("puppetnode::#{name}(#{level[:name]})") do |ui|        
+            CORL.ui_group("puppetnode::#{name}(#{CORL.yellow(level[:name])})", :cyan) do |ui|        
               ui.send(level[:send], str)
             end
           end
@@ -116,23 +116,24 @@ class Puppetnode < CORL.plugin_class(:provisioner)
   # Provisioner interface operations
   
   def build(options = {})
-    super do |locations, package_info, init_location|
-      init_location.call(:modules, nil)
+    super do |locations, package_info|
+      success = true
       
-      init_location.call(:profiles, :pp)
-      init_location.call(:default, :pp)
-        
       # Build modules
-      
       locations[:module] = {}
       
       init_profile = lambda do |package_name, profile_name, profile_info|
-        package_id     = id(package_name)
-        base_directory = File.join(locations[:modules], package_id.to_s, profile_name.to_s)
+        package_id      = id(package_name)
+        base_directory  = File.join(locations[:build], 'modules', package_id.to_s, profile_name.to_s)
+        profile_success = true
+        
+        ui.info("Building CORL profile #{blue(profile_name)} modules into #{green(base_directory)}")
         
         if profile_info.has_key?(:modules)
           profile_info[:modules].each do |module_name, module_reference|
             module_directory = File.join(base_directory, module_name.to_s)
+            
+            ui.info("Building Puppet module #{blue(module_name)} at #{purple(module_reference)} into #{green(module_directory)}")
                 
             module_project = CORL.project(extended_config(:puppet_module, {
               :directory => File.join(build_directory, module_directory),
@@ -140,22 +141,34 @@ class Puppetnode < CORL.plugin_class(:provisioner)
               :create    => true,
               :pull      => true
             }))
-            raise unless module_project                
+            unless module_project
+              ui.warn("Puppet module #{cyan(module_name)} failed to initialize")
+              profile_success = false
+              break
+            end             
           end
-          locations[:module][profile_id(package_name, profile_name)] = base_directory
+          locations[:module][profile_id(package_name, profile_name)] = base_directory if profile_success
+          profile_success
         end
       end     
       
       hash(package_info.get([ :provisioners, plugin_provider ])).each do |package_name, info|
         if info.has_key?(:profiles)
           info[:profiles].each do |profile_name, profile_info|
-            init_profile.call(package_name, profile_name, profile_info)
+            unless init_profile.call(package_name, profile_name, profile_info)
+              success = false
+              break
+            end
           end
         end
       end
       profiles.each do |profile_name, profile_info|
-        init_profile.call(plugin_name, profile_name, profile_info)
-      end
+        unless init_profile.call(plugin_name, profile_name, profile_info)
+          success = false
+          break  
+        end
+      end      
+      success
     end     
   end
   
@@ -194,37 +207,38 @@ class Puppetnode < CORL.plugin_class(:provisioner)
         classes = {}
             
         locations[:package].each do |name, package_directory|
-          gateway       = File.join(build_directory, package_directory, "#{type}.pp")
+          type_gateway  = File.join(build_directory, package_directory, "#{type}.pp")
           resource_name = concatenate([ name, type ])
         
           add_search_path(type, resource_name) if add_search_path
         
-          if File.exists?(gateway)
-            import(gateway)
+          if File.exists?(type_gateway)
+            import(type_gateway)
             classes[resource_name] = parameters                 
           end
         
-          directory = File.join(build_directory, package_directory, type.to_s)
-          Dir.glob(File.join(directory, '*.pp')).each do |file|
+          type_directory = File.join(build_directory, package_directory, type.to_s)
+          Dir.glob(File.join(type_directory, '*.pp')).each do |file|
             resource_name = concatenate([ name, type, File.basename(file).gsub('.pp', '') ])
             import(file)
             classes[resource_name] = parameters
           end        
         end
     
-        gateway       = File.join(build_directory, locations[:build], "#{type}.pp")
+        type_gateway  = File.join(directory, "#{type}.pp")
         resource_name = concatenate([ plugin_name, type ])
       
         add_search_path(type, resource_name) if add_search_path
      
-        if File.exists?(gateway)
-          import(gateway)
+        if File.exists?(type_gateway)
+          import(type_gateway)
           classes[resource_name] = parameters                
         end
     
-        if locations.has_key?(type)
-          directory = File.join(build_directory, locations[type])
-          Dir.glob(File.join(directory, '*.pp')).each do |file|
+        type_directory = File.join(directory, type.to_s)
+        
+        if File.directory?(type_directory)
+          Dir.glob(File.join(type_directory, '*.pp')).each do |file|
             resource_name = concatenate([ plugin_name, type, File.basename(file).gsub('.pp', '') ])
             import(file)
             classes[resource_name] = parameters
@@ -249,7 +263,7 @@ class Puppetnode < CORL.plugin_class(:provisioner)
           profiles.each do |profile|
             classes[profile.to_s] = { :require => 'Anchor[profile_start]' }
           end
-        
+          
           # Compile catalog
           node.classes = classes
           compiler.compile

data/lib/core/plugin/network.rb

@@ -4,6 +4,7 @@ module Plugin
 class Network < CORL.plugin_class(:base)
   
   init_plugin_collection 
+  task_class TaskThread
   
   #-----------------------------------------------------------------------------
   # Cloud plugin interface
@@ -133,7 +134,7 @@ class Network < CORL.plugin_class(:base)
     local_node = node_by_ip(ip_address, require_new)
         
     if local_node.nil?
-      name       = Util::Data.ensure_value(lookup(:hostname), ip_address)    
+      name       = Util::Data.ensure_value(lookup(:fqdn), ip_address)    
       local_node = CORL.node(name, extended_config(:local_node).import({ :meta => { :parent => myself }}), :local) 
     else
       local_node.network = myself

data/lib/core/plugin/node.rb

@@ -4,6 +4,7 @@ module Plugin
 class Node < CORL.plugin_class(:base)
   
   include Celluloid
+  task_class TaskThread
  
   #-----------------------------------------------------------------------------
   # Node plugin interface
@@ -11,13 +12,15 @@ class Node < CORL.plugin_class(:base)
   def normalize(reload)
     super
     
+    @class_color = :purple
+    
     export.each do |name, value|
       myself[name] = value
     end
     
     yield if block_given? # Chance to create a machine to feed hostname
     
-    ui.resource = hostname
+    ui.resource = Util::Console.colorize(hostname, @class_color)
     logger      = hostname
     
     myself[:settings] = [ "all", plugin_provider.to_s, plugin_name.to_s ] | setting(:settings, [], :array)
@@ -162,7 +165,7 @@ class Node < CORL.plugin_class(:base)
     hostname = myself[:hostname]
     
     if hostname.to_s != ui.resource.to_s 
-      ui.resource = hostname
+      ui.resource = Util::Console.colorize(hostname, @class_color)
       logger      = hostname
     end
     hostname
@@ -221,7 +224,7 @@ class Node < CORL.plugin_class(:base)
   
   def private_key
     config_key = myself[:private_key]
-    return File.expand_path(config_key) if config_key
+    return File.expand_path(config_key, network.directory) if config_key
     nil
   end
   
@@ -233,7 +236,7 @@ class Node < CORL.plugin_class(:base)
   
   def public_key
     config_key = myself[:public_key]
-    return File.expand_path(config_key) if config_key
+    return File.expand_path(config_key, network.directory) if config_key
   end
   
   #---
@@ -367,24 +370,33 @@ class Node < CORL.plugin_class(:base)
   # Machine operations
   
   def build(options = {})
-    config = Config.ensure(options)
-    info   = provisioner_info
+    config  = Config.ensure(options)
+    success = true
     
     provisioners.each do |provider, collection|
-      provider_info = info[provider]
-      profiles      = provider_info[:profiles]
+      ui.info("Building #{provider} provisioner collection")
           
       collection.each do |name, provisioner|
-        provisioner.build(options)
+        ui.info("Building #{name} provisioner")
+        
+        unless provisioner.build(options)
+          success = false
+          break
+        end
       end
     end
     
-    myself[:build] = Time.now.to_s
+    if success
+      ui.success("Saving successful build")
+      
+      myself[:build] = Time.now.to_s
     
-    save(extended_config(:build, {
-      :message => config.get(:message, "Built #{plugin_provider} node: #{plugin_name}"),
-      :remote  => config.get(:remote, :edit)  
-    }))
+      success = save(extended_config(:build, {
+        :message => config.get(:message, "Built #{plugin_provider} node: #{plugin_name}"),
+        :remote  => config.get(:remote, :edit)  
+      }))
+    end
+    success
   end
   
   #---
@@ -624,9 +636,9 @@ class Node < CORL.plugin_class(:base)
           results = active_machine.exec(commands, config.export) do |type, command, data|
             unless local?
               if type == :error
-                alert(data.gsub(/^\[[^\]]+\]\s*/, ''))
+                alert(data.gsub(/^(\e\[\d\d?m)?\[[^\]]+\]\s*/, '\\1'))
               else
-                render(data.gsub(/^\[[^\]]+\]\s*/, ''))
+                render(data.gsub(/^(\e\[\d\d?m)?\[[^\]]+\]\s*/, '\\1'))
               end
             end
             yield(:progress, { :type => type, :command => command, :data => data }) if block_given?   

data/lib/core/plugin/provisioner.rb

@@ -38,7 +38,7 @@ class Provisioner < CORL.plugin_class(:base)
   end
   
   def directory
-    myself[:directory]
+    File.join(network.directory, myself[:directory])
   end
   
   #---
@@ -136,89 +136,79 @@ class Provisioner < CORL.plugin_class(:base)
   #---
   
   def build(options = {})
-    config = Config.ensure(options)
-    
-    locations    = Config.new({ :package => {} })
+    config       = Config.ensure(options)
+    success      = true    
+    locations    = Config.new({ :build => id.to_s, :package => {} })
     package_info = Config.new
     
-    init_location = lambda do |name, ext = '', base_directory = nil|
-      name           = name.to_sym      
-      base_directory = directory if base_directory.nil?
-      build_base     = build_directory
-          
-      # Create directory
-      locations[name] = File.join(locations[:build], name.to_s)        
-      FileUtils.mkdir_p(File.join(build_base, locations[name]))
-     
-      # Copy directory contents
-      unless ext.nil?
-        ext             = ! ext.to_s.empty? ? ".#{ext}" : ''
-        local_directory = File.join(base_directory, name.to_s)
-        
-        if File.directory?(local_directory)  
-          unless Dir.glob(File.join(local_directory, "*#{ext}")).empty?            
-            FileUtils.cp_r(local_directory, File.join(build_base, locations[:build]))
-          end
-        end
-        
-        # Copy gateway file
-        gateway_file = File.join(base_directory, "#{name}#{ext}")
-      
-        if File.exists?(gateway_file)
-          FileUtils.cp(gateway_file, File.join(build_base, locations[:build]))
-        end
-      end  
-    end
-    
     init_package = lambda do |name, reference|
-      package_directory = File.join(locations[:packages], id(name).to_s)
+      package_directory = File.join(locations[:build], 'packages', id(name).to_s)
+      package_success   = true
+      
+      ui.info("Building package #{blue(name)} at #{purple(reference)} into #{green(package_directory)}")
         
       project = CORL.configuration(extended_config(:package, {
         :directory => File.join(build_directory, package_directory),
         :url       => reference,
         :create    => true
       }))
-      raise unless project
+      unless project
+        ui.warn("Project #{cyan(name)} failed to initialize")
+        package_success = false
+      end
       
-      package_info.import(project.export)
-      locations[:package][name] = package_directory
+      if package_success
+        package_info.import(project.export)
+        locations[:package][name] = package_directory
       
-      if project.get([ :provisioners, plugin_provider ], false)
-        project.get_hash([ :provisioners, plugin_provider ]).each do |prov_name, info|
-          if info.has_key?(:packages)
-            info[:packages].each do |sub_name, sub_reference|
-              init_package.call(sub_name, sub_reference)
+        if project.get([ :provisioners, plugin_provider ], false)
+          project.get_hash([ :provisioners, plugin_provider ]).each do |prov_name, info|
+            if info.has_key?(:packages)
+              info[:packages].each do |sub_name, sub_reference|
+                unless init_package.call(sub_name, sub_reference)
+                  package_success = false
+                  break
+                end
+              end
             end
           end
         end
       end
+      package_success
     end
     
-    locations[:build]     = id.to_s
     local_build_directory = File.join(build_directory, locations[:build])   
     
     FileUtils.rm_rf(local_build_directory)
     FileUtils.mkdir_p(local_build_directory)
     
-    init_location.call(:packages, nil)
-    
     # Build packages
     packages.each do |name, reference|
-      init_package.call(name, reference)
+      unless init_package.call(name, reference)
+        success = false
+        break
+      end
     end
-     
-    yield(locations, package_info, init_location) if block_given?
     
-    myself[:build_locations] = locations.export
-    myself[:build_info]      = package_info.get([ :provisioners, plugin_provider ])
-    myself[:build_profiles]  = find_profiles
+    if success
+      # Build provider specific components 
+      success = yield(locations, package_info) if block_given?
     
-    network.save(config.import({
-      :commit      => true,
-      :allow_empty => true, 
-      :message     => config.get(:message, "Built #{plugin_provider} provisioner #{plugin_name}"),
-      :remote      => config.get(:remote, :edit)
-    }))
+      if success
+        # Save the updates (should build the same on all nodes in same system)
+        myself[:build_locations] = locations.export
+        myself[:build_info]      = package_info.get([ :provisioners, plugin_provider ])
+        myself[:build_profiles]  = find_profiles
+    
+        success = network.save(config.import({
+          :commit      => true,
+          :allow_empty => true, 
+          :message     => config.get(:message, "Built #{plugin_provider} provisioner #{plugin_name}"),
+          :remote      => config.get(:remote, :edit)
+        }))
+      end
+    end
+    success
   end
   
   #---
@@ -232,16 +222,10 @@ class Provisioner < CORL.plugin_class(:base)
   
   def provision(profiles, options = {})
     config  = Config.ensure(options)
-    
-    use_colors               = Util::Console.use_colors
-    Util::Console.use_colors = config.get(:color, true)
-    
     success = yield(config) if block_given?
+    
     Config.save_properties(Config.get_options(:corl_log)) if success
     success
-    
-  ensure
-    Util::Console.use_colors = use_colors  
   end
        
   #-----------------------------------------------------------------------------
@@ -310,7 +294,7 @@ class Provisioner < CORL.plugin_class(:base)
         str.to_s.split('__')  
       end.flatten
     else
-      components = [ components.to_s ]
+      components = [ components.to_s.split('__') ].flatten
     end
     
     if capitalize

data/locales/en.yml

@@ -152,11 +152,12 @@ en:
             Project reference %{value} failed to parse or provider %{provider} isn't loaded >> Possible providers: %{choices}
         start: |-
           Now seeding CORL node
+      build:
+        start:
+          Building provisioner project
       provision:
         options:
           dry_run: |-
             Whether or not to build the provisioner catalog without configuring the system (default %{default_value})
-          color: |-
-            Whether or not to output log messages in color (default %{default_value})
         start: |-
           Starting provision run

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: corl
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-19 00:00:00.000000000 Z
+date: 2014-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nucleon
-  requirement: &13324860 !ruby/object:Gem::Requirement
+  requirement: &3809680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '0.1'
   type: :runtime
   prerelease: false
-  version_requirements: *13324860
+  version_requirements: *3809680
 - !ruby/object:Gem::Dependency
   name: fog
-  requirement: &13324220 !ruby/object:Gem::Requirement
+  requirement: &3808700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '1.20'
   type: :runtime
   prerelease: false
-  version_requirements: *13324220
+  version_requirements: *3808700
 - !ruby/object:Gem::Dependency
   name: unf
-  requirement: &13323500 !ruby/object:Gem::Requirement
+  requirement: &3807080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '0.1'
   type: :runtime
   prerelease: false
-  version_requirements: *13323500
+  version_requirements: *3807080
 - !ruby/object:Gem::Dependency
   name: facter
-  requirement: &13320840 !ruby/object:Gem::Requirement
+  requirement: &3821840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '1.7'
   type: :runtime
   prerelease: false
-  version_requirements: *13320840
+  version_requirements: *3821840
 - !ruby/object:Gem::Dependency
   name: hiera
-  requirement: &13320140 !ruby/object:Gem::Requirement
+  requirement: &3820160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: '1.3'
   type: :runtime
   prerelease: false
-  version_requirements: *13320140
+  version_requirements: *3820160
 - !ruby/object:Gem::Dependency
   name: puppet
-  requirement: &13319540 !ruby/object:Gem::Requirement
+  requirement: &3830860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: '3.2'
   type: :runtime
   prerelease: false
-  version_requirements: *13319540
+  version_requirements: *3830860
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &13319060 !ruby/object:Gem::Requirement
+  requirement: &3830020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,10 @@ dependencies:
         version: '1.2'
   type: :development
   prerelease: false
-  version_requirements: *13319060
+  version_requirements: *3830020
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &13318580 !ruby/object:Gem::Requirement
+  requirement: &3829000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +98,10 @@ dependencies:
         version: '2.0'
   type: :development
   prerelease: false
-  version_requirements: *13318580
+  version_requirements: *3829000
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &13944680 !ruby/object:Gem::Requirement
+  requirement: &3828100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,10 +109,10 @@ dependencies:
         version: '2.10'
   type: :development
   prerelease: false
-  version_requirements: *13944680
+  version_requirements: *3828100
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &13804500 !ruby/object:Gem::Requirement
+  requirement: &3827260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,10 +120,10 @@ dependencies:
         version: '3.12'
   type: :development
   prerelease: false
-  version_requirements: *13804500
+  version_requirements: *3827260
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &13627500 !ruby/object:Gem::Requirement
+  requirement: &3826400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -131,7 +131,7 @@ dependencies:
         version: '0.8'
   type: :development
   prerelease: false
-  version_requirements: *13627500
+  version_requirements: *3826400
 description: Framework that provides a simple foundation for growing organically in
   the cloud
 email: adrian.webb@coralnexus.com