corl 0.4.16 → 0.4.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b4edb990285aa696910d5e592213b5a91b5e5f6e
-  data.tar.gz: de66aaa2bdf8677c6a0eafead3233557326b0595
+  metadata.gz: 7f85f65c16fb13179e2a8b095eb32405cc380d0e
+  data.tar.gz: 03aafb7bbc09d7430714188aa08eaf38a84952bd
 SHA512:
-  metadata.gz: 953891d538e823b0dd7c29b2b07ca2311a3abce4aab3cb69771e61b17c6bef119d02c87732cc1a84bdc059692c45141b7c78751d88bd4efdd0344faa5c4a2fb6
-  data.tar.gz: c92c87e4c910691a82d140ac025d381db9b7cb691f62f024ca4598b8bece484dcfafbcbfd85cb9e63ae2b1c3aee362275ac03aa3cf77404e98d463525c658d76
+  metadata.gz: f35af71d60106c4fb0aef663b7b672fe500699f57daddc583cca739fffb6f3762b9700a85da78df842230b5e01cb1c561f80ddb5a141ffc66158d2b20aedc998
+  data.tar.gz: 1192b85dc7fd5397b4cb2ffb2dd5736b54dddf3539be45005688476e3f81f8babf24fb700552919ef4004b2cd1de1a6f51f0a395041b7c0ad7143be8edebb221

data/Gemfile.lock

@@ -1,9 +1,9 @@
 GIT
   remote: git://github.com/coralnexus/nucleon.git
-  revision: 924063f5b5fd6680e438d7b3ea91bc655e7e6837
+  revision: f665008257c0307125812676a84ebe2820e05398
   branch: 0.1
   specs:
-    nucleon (0.1.13)
+    nucleon (0.1.14)
       celluloid (~> 0.15)
       childprocess (~> 0.5)
       deep_merge (~> 1.0)
@@ -17,7 +17,7 @@ GIT
 
 GIT
   remote: git://github.com/mitchellh/vagrant.git
-  revision: e8f1f301e6bab643ae652d2ed1fcee0c6ee820c4
+  revision: ec35190cfc920dc816b446f1845060d819d5a7c2
   branch: master
   specs:
     vagrant (1.5.3.dev)
@@ -25,7 +25,7 @@ GIT
       childprocess (~> 0.5.0)
       erubis (~> 2.7.0)
       i18n (~> 0.6.0)
-      listen (~> 2.4.0)
+      listen (~> 2.7.1)
       log4r (~> 1.1.9, < 1.1.11)
       net-scp (~> 1.1.0)
       net-ssh (>= 2.6.6, < 2.8.0)
@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    corl (0.4.16)
+    corl (0.4.17)
       facter (~> 1.7)
       fog (~> 1.20)
       hiera (~> 1.3)
@@ -50,6 +50,9 @@ GEM
     builder (3.2.2)
     celluloid (0.15.2)
       timers (~> 1.1.0)
+    celluloid-io (0.15.0)
+      celluloid (>= 0.15.0)
+      nio4r (>= 0.5.0)
     childprocess (0.5.2)
       ffi (~> 1.0, >= 1.0.11)
     deep_merge (1.0.1)
@@ -67,7 +70,7 @@ GEM
       fog-core (~> 1.21, >= 1.21.1)
       fog-json
       nokogiri (~> 1.5, >= 1.5.11)
-    fog-brightbox (0.0.1)
+    fog-brightbox (0.0.2)
       fog-core
       fog-json
     fog-core (1.21.1)
@@ -89,7 +92,7 @@ GEM
       multi_json (>= 1.7.5, < 2.0)
       nokogiri (~> 1.6.0)
       oauth2
-    hashie (2.1.0)
+    hashie (2.1.1)
     hiera (1.3.2)
       json_pure
     highline (1.6.21)
@@ -107,8 +110,9 @@ GEM
     json_pure (1.8.1)
     jwt (0.1.11)
       multi_json (>= 1.5)
-    listen (2.4.1)
+    listen (2.7.1)
       celluloid (>= 0.15.2)
+      celluloid-io (>= 0.15.0)
       rb-fsevent (>= 0.9.3)
       rb-inotify (>= 0.9)
     log4r (1.1.10)
@@ -121,6 +125,7 @@ GEM
       net-ssh (>= 2.6.5)
     net-ssh (2.7.0)
     netrc (0.7.7)
+    nio4r (1.0.0)
     nokogiri (1.6.1)
       mini_portile (~> 0.5.0)
     oauth2 (0.9.3)

data/VERSION

@@ -1 +1 @@
-0.4.16
+0.4.17

data/bootstrap/os/ubuntu/01_git.sh

@@ -4,11 +4,9 @@
 # Install Git.
 apt-get -y install git || exit 10
 
-echo "Hello $USER"
-
 # Make sure it is easy to communicate with repo hosts
-mkdir -p ~/.ssh || exit 11
-touch ~/.ssh/known_hosts || exit 12
+mkdir -p "/root/.ssh" || exit 11
+touch "/root/.ssh/known_hosts" || exit 12
 
 ssh-keygen -R github.com 2>&1 || exit 13 # No duplicates
-ssh-keyscan -H github.com >> ~/.ssh/known_hosts || exit 14
+ssh-keyscan -H github.com >> "/root/.ssh/known_hosts" || exit 14

data/bootstrap/os/ubuntu/05_ruby.sh

@@ -2,12 +2,12 @@
 #-------------------------------------------------------------------------------
 
 # Install Ruby.
-add-apt-repository ppa:brightbox/ruby-ng-experimental 2>&1 || exit 51
+add-apt-repository -y ppa:brightbox/ruby-ng-experimental 2>&1 || exit 51
 apt-get update || exit 52
 
-apt-get -y install ruby2.1 ruby2.1-dev || exit 53
-update-alternatives --set ruby /usr/bin/ruby2.1 || exit 54
-update-alternatives --set gem /usr/bin/gem2.1 || exit 55
+apt-get -y install ruby2.0 ruby2.0-dev || exit 53
+update-alternatives --set ruby /usr/bin/ruby2.0 || exit 54
+update-alternatives --set gem /usr/bin/gem2.0 || exit 55
 
 # Set Gem options
 ( cat <<'EOP'

data/bootstrap/os/ubuntu/06_puppet.sh

@@ -13,7 +13,7 @@ chmod 0644 /etc/apt/sources.list.d/puppet.list || exit 62
 
 # Install Puppet
 apt-get -y update || exit 63
-apt-get -y install puppet || exit 64
+apt-get -y install puppet=3.4.3-1puppetlabs1 || exit 64
 
 # Set up Hiera configuration
 mkdir -p /var/corl/config || exit 65

data/corl.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: corl 0.4.16 ruby lib
+# stub: corl 0.4.17 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "corl"
-  s.version = "0.4.16"
+  s.version = "0.4.17"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Adrian Webb"]
-  s.date = "2014-04-08"
+  s.date = "2014-04-14"
   s.description = "Framework that provides a simple foundation for growing organically in the cloud"
   s.email = "adrian.webb@coralnexus.com"
   s.executables = ["corl"]
@@ -58,6 +58,7 @@ Gem::Specification.new do |s|
     "lib/CORL/action/lookup.rb",
     "lib/CORL/action/machines.rb",
     "lib/CORL/action/provision.rb",
+    "lib/CORL/action/reboot.rb",
     "lib/CORL/action/regions.rb",
     "lib/CORL/action/seed.rb",
     "lib/CORL/action/spawn.rb",
@@ -83,6 +84,7 @@ Gem::Specification.new do |s|
     "lib/core/mixin/action/keypair.rb",
     "lib/core/mixin/lookup.rb",
     "lib/core/mixin/macro/network_settings.rb",
+    "lib/core/mod/facter_loader.rb",
     "lib/core/mod/fog_aws_server.rb",
     "lib/core/mod/fog_rackspace_server.rb",
     "lib/core/mod/hiera_backend.rb",
@@ -98,6 +100,10 @@ Gem::Specification.new do |s|
     "lib/core/util/puppet/resource.rb",
     "lib/core/util/puppet/resource_group.rb",
     "lib/core/vagrant/Vagrantfile",
+    "lib/core/vagrant/action.rb",
+    "lib/core/vagrant/actions/create_shares.rb",
+    "lib/core/vagrant/actions/delete_cache.rb",
+    "lib/core/vagrant/actions/init_keys.rb",
     "lib/core/vagrant/commands/launcher.rb",
     "lib/core/vagrant/config.rb",
     "lib/core/vagrant/plugins.rb",

data/lib/CORL/action/reboot.rb

@@ -0,0 +1,59 @@
+
+module CORL
+module Action
+class Reboot < Plugin::CloudAction
+ 
+  #-----------------------------------------------------------------------------
+  # Settings
+ 
+  def configure
+    super do
+      register :reboot_nodes, :array, nil do |values|
+        if values.nil?
+          warn('corl.actions.reboot.errors.reboot_nodes_empty')
+          next false 
+        end
+        
+        node_plugins = CORL.loaded_plugins(:node)
+        success      = true
+        
+        values.each do |value|
+          if info = CORL.plugin_class(:node).translate_reference(value)
+            if ! node_plugins.keys.include?(info[:provider].to_sym) || info[:name].empty?
+              warn('corl.actions.reboot.errors.reboot_nodes', { :value => value, :node_provider => info[:provider],  :name => info[:name] })
+              success = false
+            end
+          end
+        end
+        success
+      end
+    end
+  end
+
+  #---
+  
+  def ignore
+    [ :nodes ]
+  end
+  
+  def arguments
+    [ :reboot_nodes ]
+  end
+  
+  #-----------------------------------------------------------------------------
+  # Operations
+   
+  def execute
+    super do |local_node, network|
+      ensure_network(network) do
+        batch_success = network.batch(settings[:reboot_nodes], settings[:node_provider], settings[:parallel]) do |node|
+          info('corl.actions.reboot.start', { :provider => node.plugin_provider, :name => node.plugin_name })
+          node.reload  
+        end
+        myself.status = code.batch_error unless batch_success
+      end
+    end
+  end
+end
+end
+end

data/lib/CORL/configuration/file.rb

@@ -76,7 +76,8 @@ class File < CORL.plugin_class(:configuration)
             
             generate_routes.call(config_name, parse_properties)
             properties.import(parse_properties)
-          end         
+          end
+          CORL.remove_plugin(parser) if parser         
         end
       end
     end
@@ -189,6 +190,7 @@ class File < CORL.plugin_class(:configuration)
           else
             success = false
           end
+          CORL.remove_plugin(renderer)
         else
           success = false
         end

data/lib/CORL/machine/vagrant.rb

@@ -104,16 +104,18 @@ class Vagrant < CORL.plugin_class(:machine)
   def download(remote_path, local_path, options = {})
     super do |config, success|
       begin
-        server.communicate.tap do |comm|
-          comm.download(remote_path, local_path)  
+        if init_ssh_session
+          Util::SSH.download(node.public_ip, node.user, remote_path, local_path, config.export) do |name, received, total|
+            yield(name, received, total) if block_given?
+          end
+          true
+        else
+          false
         end
-        success = true
-          
       rescue Exception => error
         ui.error(error.message)
-        success = false
+        false
       end
-      success
     end  
   end
   
@@ -122,16 +124,18 @@ class Vagrant < CORL.plugin_class(:machine)
   def upload(local_path, remote_path, options = {})
     super do |config, success|
       begin
-        server.communicate.tap do |comm|
-          comm.upload(local_path, remote_path)  
+        if init_ssh_session
+          Util::SSH.upload(node.public_ip, node.user, local_path, remote_path, config.export) do |name, sent, total|
+            yield(name, sent, total) if block_given?
+          end
+          true
+        else
+          false
         end
-        success = true
-          
       rescue Exception => error
         ui.error(error.message)
-        success = false
+        false
       end
-      success
     end  
   end
   
@@ -139,41 +143,22 @@ class Vagrant < CORL.plugin_class(:machine)
   
   def exec(commands, options = {})
     super do |config, results|
-      codes :vagrant_exec_failed
-        
-      begin
-        server.communicate.tap do |comm|            
-          commands.each do |command|
-            as_admin = command.match(/^sudo\s+/) ? true : false
-            result   = Util::Shell::Result.new(command.gsub(/^sudo\s+/, '').strip)
-            
-            result.status = comm.execute(result.command, { :sudo => as_admin }) do |type, data|
-              case type
-              when :stdout
-                result.append_output(data)
-                yield(:output, result.command, data) if block_given?      
-              when :stderr
-                result.append_errors(data)
-                yield(:error, result.command, data) if block_given?  
-              end  
-            end
-            results << result
-          end
+      if init_ssh_session
+        results = Util::SSH.exec(node.public_ip, node.user, commands) do |type, command, data|
+          yield(type, command, data) if block_given?  
         end
-          
-      rescue Exception => error
-        ui.error(error.message)
-        results << Util::Shell::Result.new(command, code.vagrant_exec_failed)
-      end      
+      else
+        results = nil
+      end
       results
-    end    
+    end
   end
   
   #---
   
   def terminal(user, options = {})
     super do |config|
-      run(:ssh, { :ssh_opts => config.export }) 
+      Util::SSH.terminal(node.public_ip, user, config.export)
     end
   end
   
@@ -181,7 +166,8 @@ class Vagrant < CORL.plugin_class(:machine)
   
   def reload(options = {})
     super do |config|
-      run(:reload, config)
+      success = run(:reload, config)
+      success = init_ssh_session(true, config.get(:tries, 5), config.get(:sleep_time, 5)) if success
     end
   end
 
@@ -200,6 +186,7 @@ class Vagrant < CORL.plugin_class(:machine)
       FileUtils.rm_f(box_path)
       
       begin
+        close_ssh_session
         success = run(:package, config.defaults({ 'package.output' => box_path }), false)
       
         node.set_cache_setting(:box, box_name)
@@ -277,6 +264,7 @@ class Vagrant < CORL.plugin_class(:machine)
           Util::Disk.delete(box_path)
         end
       end
+      close_ssh_session if success
       success
     end
   end
@@ -353,6 +341,34 @@ class Vagrant < CORL.plugin_class(:machine)
     success
   end
   protected :run
+  
+  #---
+    
+  def init_ssh_session(reset = false, tries = 5, sleep_secs = 5)
+    success = true
+        
+    begin
+      Util::SSH.session(node.public_ip, node.user, node.ssh_port, node.private_key, reset)
+            
+    rescue Exception => error
+      if tries > 1
+        sleep(sleep_secs)
+        
+        tries -= 1
+        reset  = true
+        retry
+      else
+        success = false
+      end
+    end
+    success
+  end
+  
+  #---
+  
+  def close_ssh_session
+    Util::SSH.close_session(node.public_ip, node.user)
+  end
 end
 end
 end

data/lib/CORL/node/vagrant.rb

@@ -86,6 +86,7 @@ class Vagrant < CORL.plugin_class(:node)
     super do |op, config|
       if op == :config
         config.import(exec_options(:create))
+        config[:provision_enabled] = false
       end     
     end
   end
@@ -135,6 +136,7 @@ class Vagrant < CORL.plugin_class(:node)
     super do |op, config|
       if op == :config
         config.import(exec_options(:start))
+        config[:provision_enabled] = false
       end
     end
   end
@@ -145,6 +147,7 @@ class Vagrant < CORL.plugin_class(:node)
     super do |op, config|
       if op == :config
         config.import(exec_options(:reload))
+        config[:provision_enabled] = false
       end
     end
   end

data/lib/CORL/provisioner/puppetnode.rb

@@ -1,4 +1,3 @@
-    
 module CORL
 module Provisioner
 class Puppetnode < CORL.plugin_class(:provisioner)
@@ -17,26 +16,26 @@ class Puppetnode < CORL.plugin_class(:provisioner)
         Puppet::Util::Log.newdesttype id do
           def handle(msg)
             levels = {
-              :emerg   => { :name => 'emergency', :send => :error },
-              :alert   => { :name => 'alert', :send => :error },
-              :crit    => { :name => 'critical', :send => :error },
-              :err     => { :name => 'error', :send => :error },
+              :emerg => { :name => 'emergency', :send => :error },
+              :alert => { :name => 'alert', :send => :error },
+              :crit => { :name => 'critical', :send => :error },
+              :err => { :name => 'error', :send => :error },
               :warning => { :name => 'warning', :send => :warn },
-              :notice  => { :name => 'notice', :send => :success },
-              :info    => { :name => 'info', :send => :info },
-              :debug   => { :name => 'debug', :send => :info }
+              :notice => { :name => 'notice', :send => :success },
+              :info => { :name => 'info', :send => :info },
+              :debug => { :name => 'debug', :send => :info }
             }
-            str   = msg.respond_to?(:multiline) ? msg.multiline : msg.to_s
-            str   = msg.source == "Puppet" ? str : "#{CORL.blue(msg.source)}: #{str}"
+            str = msg.respond_to?(:multiline) ? msg.multiline : msg.to_s
+            str = msg.source == "Puppet" ? str : "#{CORL.blue(msg.source)}: #{str}"
             level = levels[msg.level]
         
-            CORL.ui_group("puppetnode::#{name}(#{CORL.yellow(level[:name])})", :cyan) do |ui|        
+            CORL.ui_group("puppetnode::#{name}(#{CORL.yellow(level[:name])})", :cyan) do |ui|
               ui.send(level[:send], str)
             end
           end
         end
       end
-    end  
+    end
   end
     
   #---
@@ -73,7 +72,7 @@ class Puppetnode < CORL.plugin_class(:provisioner)
     
     Puppet[:default_file_terminus] = :file_server
       
-    node = get_node    
+    node = get_node
     Puppet[:node_name_value] = id.to_s
     
     unless profiles.empty?
@@ -87,9 +86,9 @@ class Puppetnode < CORL.plugin_class(:provisioner)
       if manifest.match(/^packages\/.*/)
         manifest = File.join(build_directory, locations[:build], manifest)
       else
-        manifest = File.join(network.directory, directory, manifest)    
-      end    
-      Puppet[:manifest] = manifest   
+        manifest = File.join(network.directory, directory, manifest)
+      end
+      Puppet[:manifest] = manifest
     end
     
     @compiler = Puppet::Parser::Compiler.new(node)
@@ -102,7 +101,7 @@ class Puppetnode < CORL.plugin_class(:provisioner)
    
   def get_node
     node_id = id.to_s
-    node    = Puppet::Node.indirection.find(node_id)
+    node = Puppet::Node.indirection.find(node_id)
          
     if facts = Puppet::Node::Facts.indirection.find(node_id)
       facts.name = node_id
@@ -123,8 +122,8 @@ class Puppetnode < CORL.plugin_class(:provisioner)
       locations[:module] = {}
       
       init_profile = lambda do |package_name, profile_name, profile_info|
-        package_id      = id(package_name)
-        base_directory  = File.join(locations[:build], 'modules', package_id.to_s, profile_name.to_s)
+        package_id = id(package_name)
+        base_directory = File.join(locations[:build], 'modules', package_id.to_s, profile_name.to_s)
         profile_success = true
         
         ui.info("Building CORL profile #{blue(profile_name)} modules into #{green(base_directory)}")
@@ -139,20 +138,20 @@ class Puppetnode < CORL.plugin_class(:provisioner)
                 
             module_project = CORL.project(extended_config(:puppet_module, {
               :directory => full_module_directory,
-              :url       => module_reference,
-              :create    => File.directory?(full_module_directory) ? false : true,
-              :pull      => true
+              :url => module_reference,
+              :create => File.directory?(full_module_directory) ? false : true,
+              :pull => true
             }))
             unless module_project
               ui.warn("Puppet module #{cyan(module_name)} failed to initialize")
               profile_success = false
               break
-            end             
+            end
           end
           locations[:module][profile_id(package_name, profile_name)] = base_directory if profile_success
           profile_success
         end
-      end     
+      end
       
       hash(package_info.get([ :provisioners, plugin_provider ])).each do |package_name, info|
         if info.has_key?(:profiles)
@@ -167,35 +166,35 @@ class Puppetnode < CORL.plugin_class(:provisioner)
       profiles.each do |profile_name, profile_info|
         unless init_profile.call(plugin_name, profile_name, profile_info)
           success = false
-          break  
+          break
         end
-      end      
+      end
       success
-    end     
+    end
   end
   
   #---
   
   def lookup(property, default = nil, options = {})
-    Util::Puppet.lookup(property, default, Config.ensure(options).defaults({ 
-      :provisioner  => :puppetnode,
-      :puppet_scope => scope      
+    Util::Puppet.lookup(property, default, Config.ensure(options).defaults({
+      :provisioner => :puppetnode,
+      :puppet_scope => scope
     }))
   end
   
   #--
   
   def import(files, options = {})
-    Util::Puppet.import(files, Config.ensure(options).defaults({ 
-      :puppet_scope       => scope, 
-      :puppet_import_base => network.directory 
+    Util::Puppet.import(files, Config.ensure(options).defaults({
+      :puppet_scope => scope,
+      :puppet_import_base => network.directory
     }))
   end
   
   #---
   
   def add_search_path(type, resource_name)
-    Config.set_options([ :all, type ], { :search => [ resource_name.to_s ] })  
+    Config.set_options([ :all, type ], { :search => [ resource_name.to_s ] })
   end
    
   #---
@@ -203,20 +202,20 @@ class Puppetnode < CORL.plugin_class(:provisioner)
   def provision(profiles, options = {})
     super do |config|
       locations = build_locations
-      success   = true
+      success = true
     
       include_location = lambda do |type, parameters = {}, add_search_path = false|
         classes = {}
             
         locations[:package].each do |name, package_directory|
-          type_gateway  = File.join(build_directory, package_directory, "#{type}.pp")
+          type_gateway = File.join(build_directory, package_directory, "#{type}.pp")
           resource_name = concatenate([ name, type ])
         
           add_search_path(type, resource_name) if add_search_path
         
           if File.exists?(type_gateway)
             import(type_gateway)
-            classes[resource_name] = parameters                 
+            classes[resource_name] = parameters
           end
         
           type_directory = File.join(build_directory, package_directory, type.to_s)
@@ -224,17 +223,17 @@ class Puppetnode < CORL.plugin_class(:provisioner)
             resource_name = concatenate([ name, type, File.basename(file).gsub('.pp', '') ])
             import(file)
             classes[resource_name] = parameters
-          end        
+          end
         end
     
-        type_gateway  = File.join(directory, "#{type}.pp")
+        type_gateway = File.join(directory, "#{type}.pp")
         resource_name = concatenate([ plugin_name, type ])
       
         add_search_path(type, resource_name) if add_search_path
      
         if File.exists?(type_gateway)
           import(type_gateway)
-          classes[resource_name] = parameters                
+          classes[resource_name] = parameters
         end
     
         type_directory = File.join(directory, type.to_s)
@@ -250,11 +249,11 @@ class Puppetnode < CORL.plugin_class(:provisioner)
       end
     
       @@puppet_lock.synchronize do
-        begin          
+        begin
           ui.info("Starting catalog generation")
           
-          start_time = Time.now        
-          node       = init_puppet(profiles)
+          start_time = Time.now
+          node = init_puppet(profiles)
         
           # Include defaults
           classes = include_location.call(:default, {}, true)
@@ -289,7 +288,7 @@ class Puppetnode < CORL.plugin_class(:provisioner)
           raise error
           Puppet.log_exception(error)
         end
-      end    
+      end
       success
     end
   end