corl 0.4.26 → 0.4.27

data/README.rdoc

@@ -21,7 +21,7 @@ Note:  This library is still very early in development!
 
 ==== Copyright
 
-Licensed under GPLv3.  See LICENSE.txt for further details.
+Licensed under Apache License 2.0.  See LICENSE.txt for further details.
 
-Copyright (c) 2013  Adrian Webb <adrian.webb@coralnexus.com>
+Copyright (c) 2013-2014  Adrian Webb <adrian.webb@coralnexus.com>
 Coral Technology Group LLC

data/Rakefile

@@ -29,14 +29,14 @@ Jeweler::Tasks.new do |gem|
   gem.name                  = "corl"
   gem.homepage              = "http://github.com/coralnexus/corl"
   gem.rubyforge_project     = 'corl'
-  gem.license               = "GPLv3"
+  gem.license               = "Apache License, Version 2.0"
   gem.email                 = "adrian.webb@coralnexus.com"
   gem.authors               = ["Adrian Webb"]
-  gem.summary               = %Q{Cluster Orchestration and Research Library}
+  gem.summary               = %Q{Coral Orchestration and Research Library}
   gem.description           = %Q{Framework that provides a simple foundation for growing organically in the cloud} 
   gem.required_ruby_version = '>= 1.9.1'
   gem.has_rdoc              = true
-  gem.rdoc_options << '--title' << 'Cluster Orchestration and Research Library' <<
+  gem.rdoc_options << '--title' << 'Coral Orchestration and Research Library' <<
                       '--main' << 'README.rdoc' <<
                       '--line-numbers'
                       

data/VERSION

@@ -1 +1 @@
-0.4.26
+0.4.27

data/bootstrap/os/ubuntu/00_base.sh

@@ -15,5 +15,5 @@ apt-get update || exit 4
 apt-get -y dist-upgrade || exit 5
 
 # Install basic build packages.
-apt-get -y install build-essential libnl-dev libpopt-dev libssl-dev libcurl4-openssl-dev libxslt-dev libxml2-dev || exit 6
+apt-get -y install build-essential bindfs libnl-dev libpopt-dev libssl-dev libcurl4-openssl-dev libxslt-dev libxml2-dev || exit 6
 apt-get -y install python-software-properties unzip curl || exit 7

data/corl.gemspec

@@ -2,16 +2,14 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: corl 0.4.26 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "corl"
-  s.version = "0.4.26"
+  s.version = "0.4.27"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
   s.authors = ["Adrian Webb"]
-  s.date = "2014-05-06"
+  s.date = "2014-05-16"
   s.description = "Framework that provides a simple foundation for growing organically in the cloud"
   s.email = "adrian.webb@coralnexus.com"
   s.executables = ["corl"]
@@ -51,6 +49,7 @@ Gem::Specification.new do |s|
     "lib/CORL/action/build.rb",
     "lib/CORL/action/destroy.rb",
     "lib/CORL/action/exec.rb",
+    "lib/CORL/action/facts.rb",
     "lib/CORL/action/image.rb",
     "lib/CORL/action/images.rb",
     "lib/CORL/action/ip.rb",
@@ -60,6 +59,7 @@ Gem::Specification.new do |s|
     "lib/CORL/action/provision.rb",
     "lib/CORL/action/reboot.rb",
     "lib/CORL/action/regions.rb",
+    "lib/CORL/action/revoke.rb",
     "lib/CORL/action/seed.rb",
     "lib/CORL/action/spawn.rb",
     "lib/CORL/action/ssh.rb",
@@ -102,9 +102,9 @@ Gem::Specification.new do |s|
     "lib/core/util/puppet/resource_group.rb",
     "lib/core/vagrant/Vagrantfile",
     "lib/core/vagrant/action.rb",
-    "lib/core/vagrant/actions/create_shares.rb",
     "lib/core/vagrant/actions/delete_cache.rb",
     "lib/core/vagrant/actions/init_keys.rb",
+    "lib/core/vagrant/actions/link_network.rb",
     "lib/core/vagrant/commands/launcher.rb",
     "lib/core/vagrant/config.rb",
     "lib/core/vagrant/plugins.rb",
@@ -145,12 +145,13 @@ Gem::Specification.new do |s|
     "spec/spec_helper.rb"
   ]
   s.homepage = "http://github.com/coralnexus/corl"
-  s.licenses = ["GPLv3"]
-  s.rdoc_options = ["--title", "Cluster Orchestration and Research Library", "--main", "README.rdoc", "--line-numbers"]
+  s.licenses = ["Apache License, Version 2.0"]
+  s.rdoc_options = ["--title", "Coral Orchestration and Research Library", "--main", "README.rdoc", "--line-numbers"]
+  s.require_paths = ["lib"]
   s.required_ruby_version = Gem::Requirement.new(">= 1.9.1")
   s.rubyforge_project = "corl"
-  s.rubygems_version = "2.2.2"
-  s.summary = "Cluster Orchestration and Research Library"
+  s.rubygems_version = "2.0.14"
+  s.summary = "Coral Orchestration and Research Library"
 
   if s.respond_to? :specification_version then
     s.specification_version = 4

data/lib/CORL/action/authorize.rb

@@ -10,6 +10,7 @@ class Authorize < Plugin::CloudAction
     super do
       codes :key_store_failure
       
+      register :reset, :bool, false
       register :public_key, :str, nil
     end
   end
@@ -33,6 +34,8 @@ class Authorize < Plugin::CloudAction
         public_key      = settings[:public_key].strip
         key_found       = false
         
+        File.delete(authorized_keys) if settings[:reset]
+                
         if File.exists?(authorized_keys)
           Util::Disk.read(authorized_keys).split("\n").each do |line|
             if line.strip.include?(public_key)

data/lib/CORL/action/facts.rb

@@ -0,0 +1,24 @@
+
+module CORL
+module Action
+class Facts < Plugin::CloudAction
+ 
+  #-----------------------------------------------------------------------------
+  # Settings
+
+  #-----------------------------------------------------------------------------
+  # Operations
+   
+  def execute
+    super do |node, network|
+      ensure_node(node) do
+        facter_facts = Config.facts
+      
+        ui.info(Util::Data.to_json(facter_facts, true))
+        myself.result = facter_facts
+      end
+    end
+  end
+end
+end
+end

data/lib/CORL/action/lookup.rb

@@ -34,9 +34,10 @@ class Lookup < Plugin::CloudAction
     super do |node, network|
       ensure_node(node) do
         property = settings.delete(:property)
-        value    = node.lookup(property, nil, settings)
+        value    = lookup(property, nil, settings)
       
-        ui.info(sprintf("#{property} = %s", value.inspect))
+        ui.info(Util::Data.to_json(value, true))
+        myself.result = value
       end
     end
   end

data/lib/CORL/action/revoke.rb

@@ -0,0 +1,55 @@
+
+module CORL
+module Action
+class Revoke < Plugin::CloudAction
+ 
+  #-----------------------------------------------------------------------------
+  # Settings
+  
+  def configure
+    super do
+      codes :key_remove_failure
+      
+      register :public_key, :str, nil
+    end
+  end
+  
+  #---
+  
+  def arguments
+    [ :public_key ]
+  end
+
+  #-----------------------------------------------------------------------------
+  # Operations
+   
+  def execute
+    super do |node, network|
+      info('corl.actions.revoke.start')
+      
+      ensure_node(node) do
+        ssh_path        = Util::SSH.key_path
+        authorized_keys = File.join(ssh_path, 'authorized_keys')        
+        public_key      = settings[:public_key].strip
+        output_keys     = []
+                
+        if File.exists?(authorized_keys)
+          Util::Disk.read(authorized_keys).split("\n").each do |line|
+            if line.strip.include?(public_key)
+              key_found = true
+            else
+              output_keys << public_key
+            end            
+          end
+        end
+        unless key_found
+          unless Util::Disk.write(authorized_keys, output_keys.join("\n"))
+            myself.status = code.key_revoke_failure
+          end
+        end
+      end
+    end
+  end
+end
+end
+end

data/lib/core/facade.rb

@@ -29,7 +29,6 @@ module Facade
   end
   
   def vagrant_config(directory, config, &code)
-    require File.join(File.dirname(__FILE__), 'vagrant', 'config.rb')
     Vagrant::Config.register(directory, config, &code)
     @@vagrant_config_loaded = true
   end

data/lib/core/mixin/lookup.rb

@@ -9,19 +9,17 @@ module Lookup
   def facts
     fact_map = {}
     
-    CORL.silence do
-      Facter.list.each do |name|
-        fact_map[name] = Facter.value(name)
-      end
+    Facter.list.each do |name|
+      fact_map[name] = Facter.value(name)
     end
-    
+        
     fact_map
   end
   
+  #---
+  
   def fact(name)
-    CORL.silence do
-      Facter.value(name)
-    end
+    Facter.value(name)
   end
   
   #-----------------------------------------------------------------------------
@@ -31,25 +29,67 @@ module Lookup
   
   #---
   
-  def hiera_config
+  def hiera_override_dir
+    nil # Override if needed. (See network and node plugins)
+  end
+  
+  #---
+  
+  def hiera_facts
+    facts # Override if needed. (See node plugin)
+  end
+  
+  #---
+  
+  def hiera_configuration
     Kernel.load File.join(File.dirname(__FILE__), '..', 'mod', 'hiera_backend.rb')
     
+    backends    = CORL.value(:hiera_backends, [ "json", "yaml" ])
     config_file = CORL.value(:hiera_config_file, File.join(Hiera::Util.config_dir, 'hiera.yaml'))
     config      = {}
 
-    if config_file && File.exist?(config_file)
+    if CORL.admin? && config_file && File.exist?(config_file)
       config = Hiera::Config.load(config_file)
     end
     config[:logger] = :corl
     
-    results = CORL.config(:hiera_config, { :config => config })
-    results[:config]
+    unless config[:merge_behavior]
+      config[:merge_behavior] = "deeper"
+    end
+    
+    unless config[:backends]
+      config[:backends] = backends
+    end
+    
+    if override_dir = hiera_override_dir
+      backends.each do |backend|
+        if config[:backends].include?(backend)
+          backend = backend.to_sym
+          config[backend] = {} unless config[backend]
+          config[backend][:datadir] = override_dir
+        end
+      end
+    end
+    
+    hiera_config = CORL.config(:hiera, config)
+    loaded_facts = Util::Data.prefix('::', hiera_facts, '')
+    
+    unless loaded_facts.empty?
+      hiera_config[:hierarchy].collect! do |search| 
+        Hiera::Interpolate.interpolate(search, loaded_facts, {})
+      end
+    end
+    
+    unless config[:hierarchy]
+      config[:hierarchy] = [ "common" ]
+    end
+    hiera_config.export
   end
   
   #---
 
-  def hiera
-    @@hiera = Hiera.new(:config => hiera_config) if @@hiera.nil?
+  def hiera(reset = false)
+    @@hiera = Hiera.new(:config => hiera_configuration) if reset || @@hiera.nil?
     @@hiera
   end
   
@@ -58,8 +98,14 @@ module Lookup
       
   def config_initialized?
     ready = false
-    if CORL.admin? && hiera && network_path = fact(:corl_network)
-      ready = File.directory?(File.join(network_path, 'config')) ? true : false
+    if hiera
+      if CORL.admin?
+        if network_path = fact(:corl_network)
+          ready = File.directory?(File.join(network_path, 'config'))
+        end
+      else
+        ready = hiera_override_dir && File.directory?(hiera_override_dir)
+      end
     end
     ready
   end
@@ -100,21 +146,19 @@ module Lookup
       debug_lookup(config, property, value, "Fact lookup")
       
       unless value
-        if CORL.admin? 
-          if config_initialized?
-            # Try to find in Hiera data store (these might be security sensitive)
-            unless hiera_scope.respond_to?('[]')
-              hiera_scope = Hiera::Scope.new(hiera_scope)
-            end
-            value = hiera.lookup(property.to_s, nil, hiera_scope, override, context)
-            debug_lookup(config, property, value, "Hiera lookup")
-          end 
-    
-          if provisioner && value.nil?
-            # Search the provisioner scope (only admins can provision a machine)
-            value = CORL.provisioner({ :name => :lookup }, provisioner).lookup(property, default, config)
-            debug_lookup(config, property, value, "Provisioner lookup")
+        if config_initialized?
+          # Try to find in Hiera data store
+          unless hiera_scope.respond_to?('[]')
+            hiera_scope = Hiera::Scope.new(hiera_scope)
           end
+          value = hiera.lookup(property.to_s, nil, hiera_scope, override, context)
+          debug_lookup(config, property, value, "Hiera lookup")
+        end 
+    
+        if CORL.admin? && provisioner && value.nil?
+          # Search the provisioner scope (only admins can provision a machine)
+          value = CORL.provisioner({ :name => :lookup }, provisioner).lookup(property, default, config)
+          debug_lookup(config, property, value, "Provisioner lookup")
         end
       end
     end

data/lib/core/mixin/macro/network_settings.rb

@@ -22,8 +22,9 @@ module NetworkSettings
       network.send("#{_type}_setting", plugin_provider, plugin_name, property, default, format)
     end
   
-    define_method :search do |property, default = nil, format = false|
-      network.send("search_#{_type}", plugin_provider, plugin_name, property, default, format)
+    define_method :search do |property, default = nil, format = false, lookup_settings = true|
+      extra_groups = ( lookup_settings ? lookup_array(:settings) : [] )
+      network.send("search_#{_type}", plugin_provider, plugin_name, property, default, format, extra_groups)
     end
   
     define_method :set_setting do |property, value = nil|
@@ -77,13 +78,13 @@ module NetworkSettings
     #---
     
     define_method :add_groups do |groups|
-      myself[:settings] = array(myself[:settings]) | array(groups)
+      myself[:settings] = array(setting(:settings)) | array(groups)
     end
     
     #---
     
     define_method :remove_groups do |groups|
-      myself[:settings] = array(myself[:settings]) - array(groups)
+      myself[:settings] = array(setting(:settings)) - array(groups)
     end    
   end
 end

data/lib/core/plugin/network.rb

@@ -44,6 +44,12 @@ class Network < CORL.plugin_class(:base)
   
   #---
   
+  def hiera_override_dir
+    File.join(directory, 'config')
+  end
+  
+  #---
+  
   def home
     extension_set(:home, ( ENV['USER'] == 'root' ? '/root' : ENV['HOME'] )) 
   end
@@ -157,15 +163,15 @@ class Network < CORL.plugin_class(:base)
   def local_node(require_new = false)
     ip_address = CORL.public_ip  
     local_node = node_by_ip(ip_address, require_new)
-        
+    
     if local_node.nil?
-      name       = Util::Data.ensure_value(lookup(:fqdn), ip_address)    
+      name       = Util::Data.ensure_value(lookup(:fqdn), ip_address)
       local_node = CORL.node(name, extended_config(:local_node).import({ :meta => { :parent => myself }}), :local) 
     else
       local_node.network = myself
       local_node.normalize(true)
       local_node.localize               
-    end    
+    end
     local_node
   end
   

data/lib/core/plugin/node.rb

@@ -85,6 +85,23 @@ class Node < CORL.plugin_class(:base)
   
   #---
   
+  def hiera_override_dir
+    network.hiera_override_dir
+  end
+  
+  #---
+  
+  def hiera_facts
+    default_configs = extended_config(:hiera_default_facts, {
+      :fqdn          => hostname, 
+      :hostname      => hostname.gsub(/\..*$/, ''),
+      :corl_provider => plugin_provider.to_s
+    })
+    Config.new(lookup_facts).defaults(default_configs).export
+  end
+  
+  #---
+  
   def machine
     @machine
   end
@@ -106,26 +123,26 @@ class Node < CORL.plugin_class(:base)
   #---
   
   def id(reset = false)
-    myself[:id] = machine.plugin_name if machine && ( reset || myself[:id].nil? )
-    myself[:id]
+    myself[:id] = machine.plugin_name if machine && ( reset || setting(:id).nil? )
+    setting(:id)
   end
   
   #---
   
   def public_ip(reset = false)
-    myself[:public_ip] = machine.public_ip if machine && ( reset || myself[:public_ip].nil? )
-    myself[:public_ip]
+    myself[:public_ip] = machine.public_ip if machine && ( reset || setting(:public_ip).nil? )
+    setting(:public_ip)
   end
   
   def private_ip(reset = false)
-    myself[:private_ip] = machine.private_ip if machine && ( reset || myself[:private_ip].nil? )
-    myself[:private_ip]
+    myself[:private_ip] = machine.private_ip if machine && ( reset || setting(:private_ip).nil? )
+    setting(:private_ip)
   end
   
   #---
   
   def hostname
-    hostname = myself[:hostname]
+    hostname = setting(:hostname)
     
     if hostname.to_s != ui.resource.to_s 
       ui.resource = Util::Console.colorize(hostname, @class_color)
@@ -137,8 +154,8 @@ class Node < CORL.plugin_class(:base)
   #---
   
   def state(reset = false)
-    myself[:state] = machine.state if machine && ( reset || myself[:state].nil? )
-    myself[:state]
+    myself[:state] = machine.state if machine && ( reset || setting(:state).nil? )
+    setting(:state)
   end
   
   #---
@@ -276,12 +293,22 @@ class Node < CORL.plugin_class(:base)
   
   #---
   
+  def custom_facts=facts
+    myself[:facts] = hash(facts)
+  end
+  
+  def custom_facts
+    search(:facts, {}, :hash, false)
+  end
+  
+  #---
+  
   def profiles=profiles
     myself[:profiles] = array(profiles)
   end
   
   def profiles
-    array(myself[:profiles])
+    [ array(myself[:profiles]), lookup_array(:profiles).reverse ].flatten
   end
   
   #---
@@ -335,8 +362,8 @@ class Node < CORL.plugin_class(:base)
   # Settings groups
   
   def machine_config
-    name   = myself[:id]
-    name   = myself[:hostname] if name.nil? || name.empty?
+    name   = setting(:id)
+    name   = setting(:hostname) if name.nil? || name.empty?
     config = Config.new({ :name => name })
     
     yield(config) if block_given?
@@ -755,6 +782,49 @@ class Node < CORL.plugin_class(:base)
     status == code.success
   end
   
+  #---
+  
+  def lookup_facts
+    if ! local? && bootstrap_script
+      result = run.facts
+      
+      if result.status == code.success
+        return Util::Data.symbol_map(Util::Data.parse_json(result.output))
+      end
+    end
+    local? ? facts : custom_facts
+  end
+  
+  #---
+  
+  def lookup_config(property, default = nil, options = {})
+    if ! local? && bootstrap_script
+      config = Config.ensure(options).import({ :property => property })
+      result = run.lookup(config)
+    
+      if result.status == code.success
+        return Util::Data.value(Util::Data.parse_json(result.output), default)
+      end
+      return default
+    end
+    options[:hiera_scope] = Util::Data.prefix('::', lookup_facts, '')
+    lookup(property, default, options)  
+  end
+  
+  #---
+  
+  def lookup_array(property, default = [], options = {})
+    config = Config.ensure(options).import({ :context => :array })
+    lookup_config(property, default, config)  
+  end
+  
+  #---
+  
+  def lookup_hash(property, default = {}, options = {})
+    config = Config.ensure(options).import({ :context => :hash })
+    lookup_config(property, default, config)  
+  end
+  
   #---
    
   def bootstrap(local_path, options = {})

data/lib/core/vagrant/actions/{create_shares.rb → link_network.rb}

@@ -2,11 +2,11 @@
 module VagrantPlugins
 module CORL
 module Action
-class CreateShares < BaseAction
+class LinkNetwork < BaseAction
 
   def call(env)
     super do
-      env[:ui].info I18n.t("corl.vagrant.actions.create_shares.start")
+      env[:ui].info I18n.t("corl.vagrant.actions.link_network.start")
       
       vm.communicate.tap do |comm|
         # TODO: Figure out a better solution for remote network path.

data/lib/core/vagrant/config.rb

@@ -240,20 +240,39 @@ module Config
   #---
   
   def self.configure_shares(node, machine)
-    success = true
+    bindfs_installed = Gems.exist?('vagrant-bindfs')
+    success          = true
+    
+    if bindfs_installed
+      machine.vm.synced_folder ".", "/vagrant", disabled: true
+      machine.vm.synced_folder ".", "/tmp/vagrant", :type => "nfs"
+      machine.bindfs.bind_folder "/tmp/vagrant", "/vagrant"
+    end
     
     Util::Data.hash(node.shares).each do |name, options|
-      config     = CORL::Config.ensure(options)      
+      config     = CORL::Config.ensure(options)
+      share_type = config.get(:type, nil)      
       local_dir  = config.delete(:local, '')
       remote_dir = config.delete(:remote, '')
       
       config.init(:create, true)
       
       unless local_dir.empty? || remote_dir.empty?
-        share_options = {}
+        bindfs_options = config.delete(:bindfs, {})
+        share_options  = {}
       
         config.keys.each do |key|
           share_options[key] = Util::Data.value(config[key])
+        end      
+        
+        if share_type && share_type.to_sym == :nfs && bindfs_installed
+          final_dir  = remote_dir
+          remote_dir = [ '/tmp', remote_dir.sub(/^\//, '') ].join('/')
+          
+          #dbg(remote_dir, 'vm bindfs local')
+          #dbg(final_dir, 'vm bindfs remote')
+          #dbg(bindfs_options, 'vm bindfs options')  
+          machine.bindfs.bind_folder remote_dir, final_dir, bindfs_options
         end
         
         #dbg(local_dir, 'vm share local')

data/lib/core/vagrant/plugins.rb

@@ -17,6 +17,7 @@ class Plugin < ::Vagrant.plugin('2')
   
   #--- 
   
+  nucleon_require(@@directory, :config)
   nucleon_require(@@directory, :action)
   nucleon_require(@@command_dir, :launcher)  
 
@@ -41,9 +42,9 @@ class Plugin < ::Vagrant.plugin('2')
     hook.after Vagrant::Action::Builtin::WaitForCommunicator, Action::InitKeys
   end
   
-  action_hook 'create-shares', :machine_action_up do |hook|
-    nucleon_require(@@action_dir, :create_shares)
-    hook.after Action::InitKeys, Action::CreateShares
+  action_hook 'link-network', :machine_action_up do |hook|
+    nucleon_require(@@action_dir, :link_network)
+    hook.after Action::InitKeys, Action::LinkNetwork
   end
   
   action_hook 'delete-cache', :machine_action_destroy do |hook|