corl 0.4.1 → 0.4.2

data/.gitmodules

@@ -1,4 +1,4 @@
 
 [submodule "bootstrap"]
 	path = bootstrap
-	url = git://github.com/coralnexus/coral-bootstrap.git
+	url = git://github.com/coralnexus/corl-bootstrap.git

data/Gemfile

@@ -2,8 +2,8 @@ source "http://rubygems.org"
 
 gem "nucleon", "~> 0.1"
 
-gem "sshkey", "~> 1.6"
 gem "fog", "~> 1.20"
+gem "unf", "~> 0.1"
 
 gem "facter", "~> 1.7"
 gem "hiera", "~> 1.3"

data/Gemfile.lock

@@ -5,6 +5,8 @@ GEM
     builder (3.2.2)
     celluloid (0.15.2)
       timers (~> 1.1.0)
+    childprocess (0.5.1)
+      ffi (~> 1.0, >= 1.0.11)
     deep_merge (1.0.1)
     descendants_tracker (0.0.3)
     diff-lcs (1.2.5)
@@ -12,6 +14,7 @@ GEM
     facter (1.7.5)
     faraday (0.9.0)
       multipart-post (>= 1.2, < 3)
+    ffi (1.9.3)
     fog (1.20.0)
       builder
       excon (~> 0.31.0)
@@ -36,9 +39,9 @@ GEM
       mime-types (~> 1.15)
       posix-spawn (~> 0.3.6)
     hashie (2.0.5)
-    hiera (1.3.1)
+    hiera (1.3.2)
       json_pure
-    highline (1.6.20)
+    highline (1.6.21)
     i18n (0.6.9)
     jeweler (2.0.1)
       builder
@@ -56,7 +59,7 @@ GEM
     log4r (1.1.10)
     mime-types (1.25.1)
     mini_portile (0.5.2)
-    multi_json (1.8.4)
+    multi_json (1.9.0)
     multi_xml (0.5.5)
     multipart-post (2.0.0)
     net-scp (1.1.2)
@@ -65,8 +68,9 @@ GEM
     netrc (0.7.7)
     nokogiri (1.6.1)
       mini_portile (~> 0.5.0)
-    nucleon (0.1.3)
+    nucleon (0.1.4)
       celluloid (~> 0.15)
+      childprocess (~> 0.5.0)
       deep_merge (~> 1.0)
       grit (~> 2.5)
       i18n (~> 0.6)
@@ -74,14 +78,14 @@ GEM
       multi_json (~> 1.7)
       netrc (~> 0.7)
       octokit (~> 2.7)
-      rgen (~> 0.6)
+      sshkey (~> 1.6)
     oauth2 (0.9.3)
       faraday (>= 0.8, < 0.10)
       jwt (~> 0.1.8)
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
       rack (~> 1.2)
-    octokit (2.7.1)
+    octokit (2.7.2)
       sawyer (~> 0.5.2)
     posix-spawn (0.3.8)
     puppet (3.4.3)
@@ -97,7 +101,7 @@ GEM
       rspec-core (~> 2.14.0)
       rspec-expectations (~> 2.14.0)
       rspec-mocks (~> 2.14.0)
-    rspec-core (2.14.7)
+    rspec-core (2.14.8)
     rspec-expectations (2.14.5)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.6)
@@ -106,6 +110,9 @@ GEM
       faraday (~> 0.8, < 0.10)
     sshkey (1.6.1)
     timers (1.1.0)
+    unf (0.1.3)
+      unf_ext
+    unf_ext (0.0.6)
     yard (0.8.7.3)
 
 PLATFORMS
@@ -121,5 +128,5 @@ DEPENDENCIES
   puppet (~> 3.2)
   rdoc (~> 3.12)
   rspec (~> 2.10)
-  sshkey (~> 1.6)
+  unf (~> 0.1)
   yard (~> 0.8)

data/VERSION

@@ -1 +1 @@
-0.4.1
+0.4.2

data/bootstrap/os/ubuntu/00_base.sh

@@ -1,5 +1,14 @@
 #!/bin/bash
 #-------------------------------------------------------------------------------
 
+# Set hostname
+echo "$HOSTNAME" > "/etc/hostname" || exit 1
+
+sed -ri 's/127\.0\.1\.1.*//' /etc/hosts
+echo "127.0.1.1 $HOSTNAME" >> /etc/hosts || exit 2
+
+# Update system packages
+apt-get update || exit 3
+
 # Install basic build packages.
-apt-get -y install build-essential || exit 1
+apt-get -y install build-essential || exit 4

data/bootstrap/os/ubuntu/05_ruby.sh

@@ -5,3 +5,9 @@
 apt-get -y install ruby1.9.1 ruby1.9.1-dev || exit 50
 update-alternatives --set ruby /usr/bin/ruby1.9.1 || exit 51
 update-alternatives --set gem /usr/bin/gem1.9.1 || exit 52
+
+# Set Gem options
+( cat <<'EOP'
+gem: --no-rdoc --no-ri 
+EOP
+) > "$HOME/.gemrc" || exit 53

data/bootstrap/os/ubuntu/06_puppet.sh

@@ -8,8 +8,8 @@ apt-key adv --recv-key --keyserver pgp.mit.edu 4BD6EC30 || exit 60
 deb http://apt.puppetlabs.com precise main dependencies
 deb-src http://apt.puppetlabs.com precise main dependencies
 EOP
-) > /etc/apt/sources.list.d/coral_puppet.list || exit 61
-chmod 0644 /etc/apt/sources.list.d/coral_puppet.list || exit 62
+) > /etc/apt/sources.list.d/puppet.list || exit 61
+chmod 0644 /etc/apt/sources.list.d/puppet.list || exit 62
 
 # Install Puppet
 apt-get -y update || exit 63
@@ -17,7 +17,7 @@ apt-get -y install puppet || exit 64
 gem install libshadow || exit 65
 
 # Set up Hiera configuration
-mkdir -p /var/coral/config || exit 66
+mkdir -p /var/corl/config || exit 66
 
 ( cat <<'EOP'
 ---
@@ -26,13 +26,13 @@ mkdir -p /var/coral/config || exit 66
   - yaml
   - json
 :yaml:
-  :datadir: /var/coral/config
+  :datadir: /var/corl/config
 :json:
-  :datadir: /var/coral/config
+  :datadir: /var/corl/config
 :hierarchy:
   - common
 EOP
 ) > /etc/hiera.yaml || exit 67
 chmod 0440 /etc/hiera.yaml || exit 68
 
-cp /etc/hiera.yaml /etc/puppet/hiera.yaml || exit 69
+ln -fs /etc/hiera.yaml /etc/puppet/hiera.yaml || exit 69

data/bootstrap/os/ubuntu/09_nucleon.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+#-------------------------------------------------------------------------------
+
+# For now we install Nucleon through the repo for testing purposes
+
+# Install the nucleon gem
+rm -Rf /tmp/nucleon
+mkdir /tmp/nucleon || exit 90
+
+cd /tmp/nucleon || exit 101
+git clone --branch 0.1 git://github.com/coralnexus/nucleon.git /tmp/nucleon || exit 91
+git submodule update --init --recursive || exit 92
+gem build /tmp/nucleon/nucleon.gemspec || exit 93
+gem install /tmp/nucleon/nucleon-*.gem || exit 94

data/bootstrap/os/ubuntu/10_corl.sh

@@ -1,9 +1,14 @@
 #!/bin/bash
 #-------------------------------------------------------------------------------
 
+# For now we install CORL through the repo for testing purposes
+
 # Install the corl gem
 rm -Rf /tmp/corl
 mkdir /tmp/corl || exit 100
 
-git clone --branch 0.4 git://github.com/coralnexus/corl.git /tmp/corl || exit 101
-gem install /tmp/corl/corl-*.gem || exit 102
+cd /tmp/corl || exit 101
+git clone --branch 0.4 git://github.com/coralnexus/corl.git /tmp/corl || exit 102
+git submodule update --init --recursive || exit 103
+gem build /tmp/corl/corl.gemspec || exit 104
+gem install /tmp/corl/corl-*.gem || exit 105

data/corl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "corl"
-  s.version = "0.4.1"
+  s.version = "0.4.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Adrian Webb"]
-  s.date = "2014-02-23"
+  s.date = "2014-03-05"
   s.description = "Framework that provides a simple foundation for growing organically in the cloud"
   s.email = "adrian.webb@coralnexus.com"
   s.executables = ["corl"]
@@ -40,45 +40,52 @@ Gem::Specification.new do |s|
     "bootstrap/os/ubuntu/01_git.sh",
     "bootstrap/os/ubuntu/05_ruby.sh",
     "bootstrap/os/ubuntu/06_puppet.sh",
+    "bootstrap/os/ubuntu/09_nucleon.sh",
     "bootstrap/os/ubuntu/10_corl.sh",
     "corl.gemspec",
+    "lib/CORL/action/authorize.rb",
     "lib/CORL/action/bootstrap.rb",
+    "lib/CORL/action/destroy.rb",
     "lib/CORL/action/exec.rb",
     "lib/CORL/action/image.rb",
     "lib/CORL/action/images.rb",
     "lib/CORL/action/lookup.rb",
     "lib/CORL/action/machines.rb",
     "lib/CORL/action/provision.rb",
+    "lib/CORL/action/regions.rb",
     "lib/CORL/action/seed.rb",
     "lib/CORL/action/spawn.rb",
+    "lib/CORL/action/ssh.rb",
     "lib/CORL/action/start.rb",
     "lib/CORL/action/stop.rb",
     "lib/CORL/configuration/file.rb",
     "lib/CORL/event/puppet.rb",
     "lib/CORL/extension/puppetloader.rb",
-    "lib/CORL/machine/fog.rb",
+    "lib/CORL/machine/aws.rb",
     "lib/CORL/machine/physical.rb",
+    "lib/CORL/machine/rackspace.rb",
     "lib/CORL/network/default.rb",
     "lib/CORL/node/aws.rb",
-    "lib/CORL/node/google.rb",
     "lib/CORL/node/local.rb",
     "lib/CORL/node/rackspace.rb",
     "lib/CORL/provisioner/puppetnode.rb",
     "lib/CORL/provisioner/puppetnode/resource.rb",
     "lib/CORL/provisioner/puppetnode/resource_group.rb",
     "lib/CORL/template/environment.rb",
+    "lib/core/errors.rb",
     "lib/core/facade.rb",
     "lib/core/mixin/action/keypair.rb",
     "lib/core/mixin/lookup.rb",
+    "lib/core/mod/fog_aws_server.rb",
     "lib/core/mod/hiera_backend.rb",
     "lib/core/plugin/action.rb",
     "lib/core/plugin/configuration.rb",
-    "lib/core/plugin/fog.rb",
+    "lib/core/plugin/fog_machine.rb",
+    "lib/core/plugin/fog_node.rb",
     "lib/core/plugin/machine.rb",
     "lib/core/plugin/network.rb",
     "lib/core/plugin/node.rb",
     "lib/core/plugin/provisioner.rb",
-    "lib/core/util/ssh.rb",
     "lib/corl.rb",
     "lib/facter/corl_config_ready.rb",
     "lib/facter/corl_exists.rb",
@@ -125,8 +132,8 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<nucleon>, ["~> 0.1"])
-      s.add_runtime_dependency(%q<sshkey>, ["~> 1.6"])
       s.add_runtime_dependency(%q<fog>, ["~> 1.20"])
+      s.add_runtime_dependency(%q<unf>, ["~> 0.1"])
       s.add_runtime_dependency(%q<facter>, ["~> 1.7"])
       s.add_runtime_dependency(%q<hiera>, ["~> 1.3"])
       s.add_runtime_dependency(%q<puppet>, ["~> 3.2"])
@@ -137,8 +144,8 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<yard>, ["~> 0.8"])
     else
       s.add_dependency(%q<nucleon>, ["~> 0.1"])
-      s.add_dependency(%q<sshkey>, ["~> 1.6"])
       s.add_dependency(%q<fog>, ["~> 1.20"])
+      s.add_dependency(%q<unf>, ["~> 0.1"])
       s.add_dependency(%q<facter>, ["~> 1.7"])
       s.add_dependency(%q<hiera>, ["~> 1.3"])
       s.add_dependency(%q<puppet>, ["~> 3.2"])
@@ -150,8 +157,8 @@ Gem::Specification.new do |s|
     end
   else
     s.add_dependency(%q<nucleon>, ["~> 0.1"])
-    s.add_dependency(%q<sshkey>, ["~> 1.6"])
     s.add_dependency(%q<fog>, ["~> 1.20"])
+    s.add_dependency(%q<unf>, ["~> 0.1"])
     s.add_dependency(%q<facter>, ["~> 1.7"])
     s.add_dependency(%q<hiera>, ["~> 1.3"])
     s.add_dependency(%q<puppet>, ["~> 3.2"])

data/lib/CORL/action/authorize.rb

@@ -0,0 +1,57 @@
+
+module CORL
+module Action
+class Authorize < Plugin::CloudAction
+ 
+  #-----------------------------------------------------------------------------
+  # Settings
+  
+  def configure
+    super do
+      codes :network_failure,
+            :key_store_failure
+      
+      register :public_key, :str, nil
+    end
+  end
+  
+  #---
+  
+  def arguments
+    [ :public_key ]
+  end
+
+  #-----------------------------------------------------------------------------
+  # Operations
+   
+  def execute
+    super do |node, network|
+      info('corl.actions.authorize.start')
+      
+      if network && node
+        ssh_path        = Util::SSH.key_path
+        authorized_keys = File.join(ssh_path, 'authorized_keys')        
+        public_key      = settings[:public_key].strip
+        key_found       = false
+        
+        if File.exists?(authorized_keys)
+          Util::Disk.read(authorized_keys).split("\n").each do |line|
+            if line.strip.include?(public_key)
+              key_found = true
+              break  
+            end            
+          end
+        end
+        unless key_found
+          unless Util::Disk.write(authorized_keys, "#{public_key}\n", { :mode => 'a' })
+            myself.status = code.key_store_failure
+          end
+        end
+      else
+        myself.status = code.network_failure
+      end
+    end
+  end
+end
+end
+end

data/lib/CORL/action/bootstrap.rb

@@ -33,6 +33,11 @@ class Bootstrap < Plugin::CloudAction
       register :bootstrap_init, :str, 'bootstrap.sh'
       
       register :bootstrap_nodes, :array, nil do |values|
+        if values.nil?
+          warn('corl.actions.bootstrap.errors.bootstrap_nodes_empty')
+          next false 
+        end
+        
         node_plugins = CORL.loaded_plugins(:node)
         success      = true
         

data/lib/CORL/action/destroy.rb

@@ -0,0 +1,64 @@
+
+module CORL
+module Action
+class Destroy < Plugin::CloudAction
+ 
+  #-----------------------------------------------------------------------------
+  # Settings
+  
+  def configure
+    super do
+      codes :network_failure
+            
+      register :destroy_nodes, :array, nil do |values|
+        if values.nil?
+          warn('corl.actions.destroy.errors.destroy_nodes_empty')
+          next false 
+        end
+        
+        node_plugins = CORL.loaded_plugins(:node)
+        success      = true
+        
+        values.each do |value|
+          if info = CORL.plugin_class(:node).translate_reference(value)
+            if ! node_plugins.keys.include?(info[:provider].to_sym) || info[:name].empty?
+              warn('corl.actions.destroy.errors.destroy_nodes', { :value => value, :node_provider => info[:provider],  :name => info[:name] })
+              success = false
+            end
+          end
+        end
+        success
+      end
+    end
+  end
+  
+  #---
+  
+  def ignore
+    [ :nodes ]
+  end
+  
+  def arguments
+    [ :destroy_nodes ]
+  end
+
+  #-----------------------------------------------------------------------------
+  # Operations
+   
+  def execute
+    super do |local_node, network|
+      info('corl.actions.destroy.start')
+      
+      if network
+        batch_success = network.batch(settings[:destroy_nodes], settings[:node_provider], settings[:parallel]) do |node|
+          node.destroy
+        end
+        myself.status = code.batch_error unless batch_success
+      else
+        myself.status = code.network_failure  
+      end
+    end
+  end
+end
+end
+end

data/lib/CORL/action/exec.rb

@@ -26,6 +26,15 @@ class Exec < Plugin::CloudAction
   def execute
     super do |node, network|
       if network && node
+        if settings[:command].length > 1
+          settings[:command].collect! do |value|
+            if value.strip.match(/\s+/)
+              value = "\"#{value}\""
+            end
+            value
+          end
+        end
+        
         command_str   = settings[:command].join(' ')
         result        = node.exec({ :commands => [ command_str ] }).first
         myself.status = result.status

data/lib/CORL/action/image.rb

@@ -8,19 +8,51 @@ class Image < Plugin::CloudAction
   
   def configure
     super do
-      codes :network_failure  
+      codes :network_failure
+      
+      register :image_nodes, :array, nil do |values|
+        if values.nil?
+          warn('corl.actions.image.errors.image_nodes_empty')
+          next false 
+        end
+        
+        node_plugins = CORL.loaded_plugins(:node)
+        success      = true
+        
+        values.each do |value|
+          if info = CORL.plugin_class(:node).translate_reference(value)
+            if ! node_plugins.keys.include?(info[:provider].to_sym) || info[:name].empty?
+              warn('corl.actions.image.errors.image_nodes', { :value => value, :node_provider => info[:provider],  :name => info[:name] })
+              success = false
+            end
+          end
+        end
+        success
+      end  
     end
   end
- 
+  
+  #---
+  
+  def ignore
+    [ :nodes ]
+  end
+  
+  def arguments
+    [ :image_nodes ]
+  end
+
   #-----------------------------------------------------------------------------
   # Operations
    
   def execute
-    super do |node, network|
-      info('corl.actions.image.start')
-      
-      if network && node
-        
+    super do |local_node, network|
+      if network
+        batch_success = network.batch(settings[:image_nodes], settings[:node_provider], settings[:parallel]) do |node|
+          info('corl.actions.image.start', { :provider => node.plugin_provider, :name => node.plugin_name })
+          node.create_image
+        end
+        myself.status = code.batch_error unless batch_success  
       else
         myself.status = code.network_failure
       end    

data/lib/CORL/action/images.rb

@@ -10,9 +10,10 @@ class Images < Plugin::CloudAction
     super do
       codes :node_load_failure,
             :image_load_failure
-            
+      
+      register :region, :str, nil      
       register :match_case, :bool, false
-      register :require_all, :bool, false
+      register :require_all, :bool, true
       register :search, :array, []
     end
   end
@@ -34,7 +35,7 @@ class Images < Plugin::CloudAction
     super do |local_node, network|
       info('corl.actions.images.start')
       
-      if node = network.test_node(settings[:node_provider])
+      if node = network.test_node(settings[:node_provider], { :region => settings[:region] })
         if images = node.images(settings[:search], settings)
           images.each do |image|
             render(node.render_image(image), { :prefix => false })

data/lib/CORL/action/lookup.rb

@@ -24,9 +24,9 @@ class Lookup < Plugin::CloudAction
   def execute
     super do |node, network|
       property = settings[:property]
-      value    = lookup(property)
+      value    = node.lookup(property)
       
-      render(sprintf("#{property} = %s", value.inspect))
+      node.render(sprintf("#{property} = %s", value.inspect))
     end
   end
 end

data/lib/CORL/action/regions.rb

@@ -0,0 +1,51 @@
+
+module CORL
+module Action
+class Regions < Plugin::CloudAction
+
+  #-----------------------------------------------------------------------------
+  # Settings
+  
+  def configure
+    super do
+      codes :node_load_failure,
+            :region_load_failure
+    end
+  end
+  
+  #---
+  
+  def ignore
+    node_ignore - [ :node_provider ]
+  end
+  
+  def arguments
+    [ :node_provider ]
+  end
+  
+  #-----------------------------------------------------------------------------
+  # Operations
+  
+  def execute
+    super do |local_node, network|
+      info('corl.actions.regions.start')
+      
+      if node = network.test_node(settings[:node_provider])
+        if regions = node.regions
+          regions.each do |region|
+            render(sprintf("> %s", region), { :prefix => false })
+          end
+          
+          myself.result = regions
+          success('corl.actions.regions.results', { :regions => regions.length }) if regions.length > 1
+        else
+          myself.status = code.region_load_failure
+        end
+      else
+        myself.status = code.node_load_failure
+      end
+    end
+  end
+end
+end
+end

data/lib/CORL/action/seed.rb

@@ -86,7 +86,7 @@ class Seed < Plugin::CloudAction
               render("Reinitializing network")
               if network = init_network
                 if network.load
-                  if node = network.local_node
+                  if node = network.local_node(true)
                     render("Updating node network configurations")
                     myself.status = code.node_save_failure unless node.save  
                   else

data/lib/CORL/action/spawn.rb

@@ -13,21 +13,20 @@ class Spawn < Plugin::CloudAction
       codes :network_failure,
             :key_failure,
             :node_create_failure
-            
+      
+      register :groups, :array, []      
       register :region, :str, nil
       register :machine_type, :str, nil
-      register :image, :str, nil      
+      register :image, :str, nil
+      register :user, :str, :root     
       register :hostnames, :array, nil
         
       keypair_config
       
-      bootstrap = CORL.action_config(:bootstrap)
-      config.defaults(bootstrap.config) if bootstrap
+      config.defaults(CORL.action_config(:bootstrap))
+      config.defaults(CORL.action_config(:seed))
       
-      if seed = CORL.action_config(:seed)
-        seed.config[:project_reference].default = "github:::coraltech/cluster-test[master]"
-        config.defaults(seed.config)
-      end
+      config[:project_reference].default = "github:::coraltech/cluster-test[master]"
     end
   end
   
@@ -52,7 +51,7 @@ class Spawn < Plugin::CloudAction
         if keypair && keypair_clean
           results       = []
           node_provider = settings.delete(:node_provider)
-                         
+          
           settings.delete(:hostnames).each do |hostname|
             if settings[:parallel]
               results << network.future.add_node(node_provider, hostname, settings)