vagabond 0.1.4 → 0.2.0

data/lib/vagabond/cookbooks/lxc/libraries/lxc.rb

@@ -224,7 +224,7 @@ class Lxc
 
   # Stop the container
   def stop
-    run_command("#{sudo}lxc-stop -n #{name}")
+    run_command("#{sudo}lxc-stop -n #{name}", :allow_failure_retry => 3)
     run_command("#{sudo}lxc-wait -n #{name} -s STOPPED", :allow_failure_retry => 2)
   end
   
@@ -276,13 +276,14 @@ class Lxc
       shlout.run_command
       shlout.error!
     rescue Mixlib::ShellOut::ShellCommandFailed, CommandFailed, Mixlib::ShellOut::CommandTimeout
-      if(args[:allow_failure])
-        true
-      elsif(retries > 0)
+      if(retries > 0)
         Chef::Log.warn "LXC run command failed: #{cmd}"
         Chef::Log.warn "Retrying command. #{args[:allow_failure_retry].to_i - retries} of #{args[:allow_failure_retry].to_i} retries remain"
+        sleep(0.3)
         retries -= 1
         retry
+      elsif(args[:allow_failure])
+        true
       else
         raise
       end

data/lib/vagabond/cookbooks/lxc/metadata.rb

@@ -5,7 +5,7 @@ description      "Chef driven Linux Containers"
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
 version          "0.1.1"
 
-supports 'ubuntu', '12.04'
+supports 'ubuntu'
 
-depends 'omnibus_updater'
+suggests 'omnibus_updater'
 suggests 'bridger'

data/lib/vagabond/cookbooks/lxc/providers/container.rb

@@ -308,7 +308,6 @@ action :clone do
   end
   
   lxc_config new_resource.name do
-    config new_resource.config
     action :create
     notifies :restart, resources(:lxc_service => "lxc config_restart[#{new_resource.name}]"), :immediately
   end

data/lib/vagabond/cookbooks/vagabond/attributes/default.rb

@@ -1,4 +1,8 @@
+default[:vagabond][:bases][:ubuntu_1004][:template] = 'ubuntu'
+default[:vagabond][:bases][:ubuntu_1004][:template_options] = {'--release' => 'lucid'}
+default[:vagabond][:bases][:ubuntu_1204][:template] = 'ubuntu'
 default[:vagabond][:bases][:ubuntu_1204][:template_options] = {'--release' => 'precise'}
+default[:vagabond][:bases][:ubuntu_1210][:template] = 'ubuntu'
 default[:vagabond][:bases][:ubuntu_1210][:template_options] = {'--release' => 'quantal'}
 default[:vagabond][:bases][:centos_58][:template] = 'centos'
 default[:vagabond][:bases][:centos_58][:template_options] = {'--release' => '5', '--releaseminor' => '8'}
@@ -10,3 +14,4 @@ default[:vagabond][:bases][:debian_6][:template] = 'debian'
 default[:vagabond][:bases][:debian_6][:create_environment] = {'SUITE' => 'squeeze'}
 default[:vagabond][:bases][:debian_7][:template] = 'debian'
 default[:vagabond][:bases][:debian_7][:create_environment] = {'SUITE' => 'wheezy'}
+default[:vagabond][:customs] = {}

data/lib/vagabond/cookbooks/vagabond/recipes/default.rb

@@ -1,8 +1,23 @@
-include_recipe 'lxc::install_dependencies'
+include_recipe 'lxc'
 
-cookbook_file '/usr/share/lxc/templates/lxc-centos' do
-  source 'lxc-centos'
-  mode 0755
+ruby_block 'LXC template: lxc-centos' do
+  block do
+    dir = %w(/usr/share /usr/lib).map do |prefix|
+      if(File.directory?(d = File.join(prefix, 'lxc/templates')))
+        d
+      end
+    end.compact.first
+    raise 'Failed to locate LXC template directory' unless dir
+    cfl = Chef::Resource::CookbookFile.new(
+      ::File.join(dir, 'lxc-centos'),
+      run_context
+    )
+    cfl.source 'lxc-centos'
+    cfl.mode 0755
+    cfl.cookbook cookbook_name.to_s
+    cfl.action :nothing
+    cfl.run_action(:create)
+  end
 end
 
 node[:vagabond][:bases].each do |name, options|
@@ -14,9 +29,9 @@ node[:vagabond][:bases].each do |name, options|
     'upgrade -y -q',
     'install curl -y -q'
   ]
-  if(%w(debian ubuntu).include?(options[:template]))
+  if(!options[:template].scan(%r{debian|ubuntu}).empty?)
     pkg_man = 'apt-get'
-  elsif(%w(fedora centos).include?(options[:template]))
+  elsif(!options[:template].scan(%r{fedora|centos}).empty?)
     pkg_man = 'yum'
   end
   if(pkg_man)
@@ -40,14 +55,22 @@ node[:vagabond][:bases].each do |name, options|
       'curl -L https://www.opscode.com/chef/install.sh | bash'
     ]
   end
+end
+
+node[:vagabond][:customs].each do |name, options|
 
+  lxc_container name do
+    action :clone
+    base_container options[:base]
+  end
+  
   if(options[:memory])
     lxc_config name do
       cgroup(
-        'memory.limit_in_bytes' => options[:memory][:maximum_ram],
+        'memory.limit_in_bytes' => options[:memory][:ram],
         'memory.memsw.limit_in_bytes' => (
-          Vagabond.get_bytes(options[:memory][:maximum_ram]) +
-          Vagabond.get_bytes(options[:memory][:maximum_swap])
+          Vagabond.get_bytes(options[:memory][:ram]) +
+          Vagabond.get_bytes(options[:memory][:swap])
         )
       )
     end

data/lib/vagabond/helpers/cheffile_loader.rb

@@ -0,0 +1,20 @@
+module Vagabond
+  class CheffileLoader
+    
+    attr_reader :cookbooks
+
+    def initialize(path=nil)
+      @cookbooks = []
+      load(path) if path
+    end
+    
+    def cookbook(name, *args)
+      cookbooks[name] = args
+    end
+
+    def load(path)
+      instance_eval(File.read(path))
+    end
+      
+  end
+end

data/lib/vagabond/helpers.rb

@@ -1,3 +1,5 @@
+require 'vagabond/constants'
+
 module Vagabond
   module Helpers
     private
@@ -11,7 +13,70 @@ module Vagabond
     end
 
     def debug(s)
-      ui.info "#{ui.color('DEBUG:', :red, :bold)} #{s}" if Config[:debug]
+      ui.info "#{ui.color('DEBUG:', :red, :bold)} #{s}" if options[:debug]
+    end
+    
+    def generated_name(n=nil)
+      n = name unless n
+      if(@_gn.nil? || @_gn[n].nil?)
+        @_gn ||= Mash.new
+        s = Digest::MD5.new
+        s << @vagabondfile.path
+        @_gn[n] = "#{n}-#{s.hexdigest}"
+      end
+      @_gn[n]
+    end
+
+    def setup_ui(ui=nil)
+      unless(ui)
+        Chef::Config[:color] = options[:color].nil? ? true : options[:color]
+        @ui = Chef::Knife::UI.new(STDOUT, STDERR, STDIN, {})
+      else
+        @ui = ui
+      end
+      options[:debug] = STDOUT if options[:debug]
+      self.class.ui = @ui
+    end
+
+    def execute
+      if(public_methods.include?(@action.to_sym))
+        send(@action)
+      else
+        ui.error "Invalid action received: #{@action}"
+        exit EXIT_CODES[:invalid_action]
+      end
+    end
+
+    def generate_hash
+      Digest::MD5.hexdigest(@vagabondfile.path)
+    end
+
+    def direct_container_command(command, args={})
+      _lxc = args[:lxc] || lxc
+      com = "#{sudo}ssh root@#{lxc.container_ip} -i /opt/hw-lxc-config/id_rsa -oStrictHostKeyChecking=no '#{command}'"
+      debug(com)
+      begin
+        cmd = Mixlib::ShellOut.new(com,
+          :live_stream => args[:live_stream] || options[:debug],
+          :timeout => args[:timeout] || 1200
+        )
+        cmd.run_command
+        cmd.error!
+        true
+      rescue
+        raise if args[:raise_on_failure]
+        false
+      end
+    end
+
+    class << self
+      def included(klass)
+        klass.class_eval do
+          class << self
+            attr_accessor :ui
+          end
+        end
+      end
     end
     
   end

data/lib/vagabond/internal_configuration.rb

@@ -1,5 +1,7 @@
 require 'digest/sha2'
+require 'json'
 require 'vagabond/helpers'
+require 'vagabond/constants'
 
 module Vagabond
   class InternalConfiguration
@@ -8,14 +10,26 @@ module Vagabond
     
     attr_reader :config
     attr_reader :ui
+    attr_reader :options
+    attr_accessor :force_bases
     
-    def initialize(v_config, ui)
-      @v_config = v_config
-      @config = Mash.new(:mappings => Mash.new)
+    def initialize(vagabondfile, ui, options, args={})
+      @vagabondfile = vagabondfile
       @checksums = Mash.new
       @ui = ui
+      @options = options
       create_store
       load_existing
+      @config = Mash.new(
+        :mappings => Mash.new,
+        :template_mappings => Mash.new,
+        :test_mappings => Mash.new
+      ).merge(config)
+      @force_bases = args[:force_bases] || []
+      ensure_state
+    end
+
+    def ensure_state
       store_checksums
       write_dna_json
       write_solo_rb
@@ -41,13 +55,15 @@ module Vagabond
             File.read(path)
           )
         )
+      else
+        @config = Mash.new
       end
     end
 
     def store_path
       FileUtils.mkdir_p(
         File.join(
-          File.dirname(@v_config.path), '.vagabond'
+          File.dirname(@vagabondfile.path), '.vagabond'
         )
       )
     end
@@ -61,26 +77,33 @@ module Vagabond
     end
 
     def write_dna_json
-      conf = Mash.new
-      @v_config.config[:boxes].map(&:last).map{|i| i[:template]}.compact.uniq.each do |t|
-        conf[t] = Mash.new(:enabled => true)
+      conf = Mash.new(:bases => Mash.new, :customs => Mash.new)
+      (Array(@vagabondfile[:nodes]).map(&:last).map{|i| i[:template]}.compact + Array(force_bases)).uniq.each do |t|
+        conf[:bases][t] = Mash.new(:enabled => true) if BASE_TEMPLATES.include?(t.to_s)
       end
-      if(@v_config.config[:templates])
-        @v_config.config[:templates].each do |t|
-          conf[t] ||= Mash.new
-          conf[t].merge!(@v_config[:templates][t])
+      Array(@vagabondfile[:templates]).each do |t_name, opts|
+        if(BASE_TEMPLATES.include?(opts[:base].to_s))
+          conf[:bases][opts[:base]] = Mash.new(:enabled => true)
+          if(opts.has_key?(:memory) && !opts[:memory].is_a?(Hash))
+            opts[:memory][:ram] = opts[:memory].to_s
+          end
+          conf[:customs][generated_name(t_name)] = opts
+          config[:template_mappings][t_name] = generated_name(t_name)
+        else
+          ui.fatal "Invalid base template encountered: #{t}"
+          ui.info ui.color("  -> Valid base templates: #{BASE_TEMPLATES.sort.join(', ')}", :red)
+          exit EXIT_CODES[:invalid_base_template]
         end
       end
       File.open(dna_path, 'w') do |file|
         file.write(
           JSON.dump(
-            :vagabond => {
-              :bases => conf
-            },
+            :vagabond => conf,
             :run_list => %w(recipe[vagabond])
           )
         )
       end
+      save
     end
 
     def write_solo_rb
@@ -106,9 +129,9 @@ module Vagabond
     end
 
     def solo_needed?
-      if(Config[:force_solo])
+      if(options[:force_solo])
         true
-      elsif(Config[:disable_solo])
+      elsif(options[:disable_solo])
         false
       else
         [dna_path, solo_path].detect do |path|
@@ -133,13 +156,14 @@ module Vagabond
     end
     
     def run_solo
-      ui.info ui.color('Ensuring expected system state (creating required template containers)', :yellow)
+      ui.info ui.color('Ensuring expected system state (creating required base containers)', :yellow)
       ui.info ui.color('   - This can take a while...', :yellow)
-      com = "#{Config[:sudo]}chef-solo -j #{File.join(store_path, 'dna.json')} -c #{File.join(store_path, 'solo.rb')}"
+      com = "#{options[:sudo]}chef-solo -j #{File.join(store_path, 'dna.json')} -c #{File.join(store_path, 'solo.rb')}"
       debug(com)
-      cmd = Mixlib::ShellOut.new(com, :timeout => 1200, :live_stream => Config[:debug])
+      cmd = Mixlib::ShellOut.new(com, :timeout => 12000, :live_stream => options[:debug])
       cmd.run_command
       cmd.error!
+      ui.info ui.color('  -> COMPLETE!', :yellow)
     end
     
     def save

data/lib/vagabond/kitchen.rb

@@ -0,0 +1,311 @@
+require 'thor'
+require 'chef'
+require 'kitchen/busser'
+
+%w(helpers vagabondfile vagabond server helpers/cheffile_loader).each do |dep|
+  require "vagabond/#{dep}"
+end
+
+module Vagabond
+  class Kitchen < Thor
+    
+    include Thor::Actions
+    include Helpers
+
+    class << self
+      def basename
+        'vagabond kitchen'
+      end
+    end
+
+    self.class_exec(&Vagabond::CLI_OPTIONS)
+    
+    attr_reader :kitchen
+    attr_reader :platform_map
+    attr_reader :vagabondfile
+    attr_reader :ui
+    attr_reader :name
+    attr_reader :action
+
+    def initialize(*args)
+      super
+    end
+
+    desc 'teardown COOKBOOK', 'Destroy containers related to COOKBOOK test'
+    method_option(:platform,
+      :type => :string,
+      :desc => 'Specify platform to destroy'
+    )
+    method_option(:suite,
+      :type => :string,
+      :desc => 'Specify suite to destroy'
+    )
+    def teardown(cookbook)
+      ui.info "#{ui.color('Vagabond:', :bold)} - Kitchen teardown for cookbook #{ui.color(name, :red)}"
+      plats = [platform || options[:platform] || platform_map.keys].flatten
+      plats.each do |plat|
+        validate_platform!(plat)
+        ui.info ui.color("  -> Tearing down platform: #{plat}", :red)
+        vagabond_instance(:destroy, plat).send(:execute)
+        ui.info ui.color("  -> Teardown of platform: #{plat} - COMPLETE!", :red)
+      end
+    end
+
+    desc 'test COOKBOOK', 'Run test kitchen on COOKBOOK'
+    method_option(:platform,
+      :type => :string,
+      :desc => 'Specify platform to test'
+    )
+    method_option(:cluster,
+      :type => :string,
+      :desc => 'Specify cluster to test'
+    )
+    method_option(:teardown,
+      :type => :boolean,
+      :default => true,
+      :desc => 'Teardown nodes automatically after testing'
+    )
+    method_option(:parallel,
+      :type => :boolean,
+      :default => false,
+      :desc => 'Build test nodes in parallel'
+    )
+    method_option(:suites,
+      :type => :string,
+      :desc => 'Specify suites to test [suite1,suite2,...]'
+    )
+    def test(cookbook)
+      setup(cookbook, :test)
+      ui.info "#{ui.color('Vagabond:', :bold)} - Kitchen testing for cookbook #{ui.color(name, :cyan)}"
+      results = Mash.new
+      platforms = [options[:platform] || platform_map.keys].flatten
+      if(options[:cluster])
+        ui.info ui.color("  -> Cluster Testing #{options[:cluster]}!", :yellow)
+        if(kitchen[:clusters].nil? || kitchen[:clusters][options[:cluster]].nil?)
+          ui.fatal "Requested cluster is not defined: #{options[:cluster]}"
+          exit EXIT_CODES[:cluster_invalid]
+        end
+        serv = Server.new
+        serv.options = options
+        serv.auto_upload # upload everything : make optional?
+        suites = kitchen[:clusters][options[:cluster]]
+        platforms.each do |platform|
+          %w(local_server_provision test destroy).each do |action|
+            suites.each do |suite_name|
+              res = self.send("#{action}_node", platform, suite_name)
+              if(action == 'test')
+                results[platform] ||=[]
+                results[platform] << {
+                  :suite_name => suite_name,
+                  :result => res
+                }
+              end
+            end
+          end
+        end
+      else
+        suites = options[:suites] ? options[:suites].split(',') : ['default']
+        platforms.each do |platform|
+          suites.each do |suite_name|
+            provision_node(platform, suite_name)
+            results[platform] ||= []
+            results[platform] << {
+              :suite_name => suite_name,
+              :result => test_node(platform, suite_name)
+            }
+            destroy_node(platform, suite_name)
+          end
+        end
+      end
+      ui.info ui.color('Kitchen Test Results:', :bold)
+      results.each do |platform, infos|
+        ui.info "  Platform: #{ui.color(platform, :blue, :bold)}"
+        infos.each do |res|
+          ui.info "    Suite: #{res[:suite_name]} -> #{res[:result] ? ui.color('SUCCESS!', :green) : ui.color('FAILED!', :red)}"
+        end
+      end
+    end
+    
+    protected
+
+    def local_server_provision_node(platform, suite_name)
+      run_list = generate_runlist(platform, suite_name)
+      v_inst = vagabond_instance(:up, platform, :suite_name => suite_name, :run_list => run_list)
+      raise "ERROR! No local chef!" unless v_inst.options[:knife_opts]
+      v_inst.send(:execute)
+    end
+    
+    # TODO: Handle failed provision!
+    def provision_node(platform, suite_name)
+      run_list = generate_runlist(platform, suite_name)
+      ui.info ui.color("  -> Provisioning suite #{suite_name} on platform: #{platform}", :cyan)
+      v_inst = vagabond_instance(:create, platform, :suite_name => suite_name)
+      v_inst.send(:execute)
+      solo_path = configure_for(v_inst.name, platform, suite_name, run_list, :dna, :cookbooks)
+      v_inst.send(:provision_solo, solo_path)
+    end
+
+    def test_node(platform, suite_name)
+      v_inst = vagabond_instance(:create, platform, :suite_name => suite_name)
+      busser = bus_node(v_inst, suite_name)
+      ui.info "#{ui.color('Kitchen:', :bold)} Running tests..."
+      cmd = busser.run_cmd
+      res = cmd.to_s.empty? ? true : v_inst.send(:direct_container_command, cmd, :live_stream => STDOUT)
+      ui.info "\n  -> #{ui.color('Testing', :bold, :cyan)} #{name} suite #{suite_name} on platform #{platform}: #{res ? ui.color('SUCCESS!', :green, :bold) : ui.color('FAILED', :red)}"
+      res
+    end
+
+    def destroy_node(platform, suite_name)
+      if(options[:teardown])
+        v_inst = vagabond_instance(:destroy, platform, :suite_name => suite_name)
+        v_inst.send(:execute)
+      end
+    end
+    
+    def setup(name, action)
+      @options = options.dup
+      @vagabondfile = Vagabondfile.new(options[:vagabond_file])
+      setup_ui
+      @name = name
+      @action = action
+      load_kitchen_yml
+    end
+
+    def configure_for(l_name, platform, suite_name, runlist, *args)
+      dir = File.join(File.dirname(vagabondfile.path), ".vagabond/node_configs/#{l_name}")
+      FileUtils.mkdir_p(dir)
+      _args = [args.include?(:integration) ? :integration : nil].compact
+      write_dna(l_name, suite_name, dir, platform, runlist, *_args) if args.include?(:dna)
+      load_cookbooks(l_name, suite_name, dir, platform, runlist, *_args) if args.include?(:cookbooks)
+      write_solo_config(dir) if args.include?(:cookbooks) && !args.include?(:integration)
+      dir
+    end
+
+    def write_solo_config(dir)
+      File.open(File.join(dir, 'solo.rb'), 'w') do |file|
+        file.write("cookbook_path '#{File.join(dir, 'cookbooks')}'\n")
+      end
+    end
+    
+    def write_dna(l_name, suite_name, dir, platform, runlist, *args)
+      key = args.include?(:integration) ? :integration_suites : :suites
+      dna = Mash.new
+      dna.merge!(platform_map[platform][:attributes] || {})
+      s_args = kitchen[:suites].detect{|s|s[:name] == suite_name}
+      if(s_args)
+        dna.merge!(s_args)
+      end
+      dna[:run_list] = runlist
+      File.open(File.join(dir, 'dna.json'), 'w') do |file|
+        file.write(JSON.dump(dna))
+      end
+    end
+
+    def cookbook_path
+      Chef::CookbookLoader.new(
+        File.join(File.dirname(vagabondfile.path), 'cookbooks')
+      ).load_cookbooks[name].root_dir
+    end
+    
+    def load_cookbooks(l_name, suite_name, dir, platform, runlist, *_args)
+      contents = ['site "http://community.opscode.com/api/v1"']
+      contents << "cookbook '#{name}', :path => '#{cookbook_path}'"
+      contents << "cookbook 'minitest-handler'"
+      # TODO - Customs from kitchen. Customs from root. Customs from cookbook
+      File.open(File.join(dir, 'Cheffile'), 'w') do |file|
+        file.write(contents.join("\n"))
+      end
+      com = "librarian-chef update"
+      debug(com)
+      c = Mixlib::ShellOut.new(com, :live_stream => options[:debug], :cwd => dir)
+      c.run_command
+      c.error!
+    end
+
+    def bus_node(v_inst, suite_name)
+      unless(::Kitchen::Busser::DEFAULT_TEST_ROOT == c_path = File.join(cookbook_path, 'test/integration'))
+        ::Kitchen::Busser.send(:remove_const, :DEFAULT_TEST_ROOT)
+        ::Kitchen::Busser.const_set(:DEFAULT_TEST_ROOT, File.join(cookbook_path, 'test/integration'))
+      end
+      busser = ::Kitchen::Busser.new(suite_name)
+      ui.info "#{ui.color('Kitchen:', :bold)} Setting up..."
+      %w(setup_cmd sync_cmd).each do |cmd|
+        com = busser.send(cmd)
+        next if com.to_s.empty?
+        v_inst.send(:direct_container_command, com)
+      end
+      busser
+    end
+    
+    def vagabond_instance(action, platform, args={})
+      options[:disable_name_validate] = true
+      v = Vagabond.new
+      v.options = options
+      v.send(:setup, action, [name, platform, args[:suite_name]].compact.join('-'),
+        :ui => ui,
+        :template => platform_map[platform][:template],
+        :disable_name_validate => true,
+        :ui => ui
+      )
+      v.internal_config.force_bases = platform_map[platform][:template]
+      v.internal_config.ensure_state
+      v.mappings_key = :test_mappings
+      v.config = Mash.new(
+        :template => platform_map[platform][:template],
+        :run_list => args[:run_list]
+      )
+      v.lxc = Lxc.new(
+        v.internal_config[v.mappings_key][v.name]
+      ) if v.internal_config[v.mappings_key][v.name]
+      v
+    end
+
+    def load_kitchen_yml
+      y_path = File.join(
+        File.dirname(vagabondfile.path), 'cookbooks', name, '.kitchen.yml'
+      )
+      if(File.exists?(y_path))
+        @kitchen = Mash.new(YAML.load(File.read(y_path)))
+      else
+        ui.fatal "Cookbook #{name} does not have a .kitchen.yml file defined!"
+        ui.info ui.color("  -> Path: #{y_path}", :red)
+        exit EXIT_CODES[:kitchen_missing_yml]
+      end
+    end
+
+    def platform_map
+      @platform_map ||= Mash.new(Hash[*(
+          kitchen[:platforms].map do |plat|
+            [
+              plat[:name], Mash.new(
+                  :template => plat[:driver_config][:box].scan(
+                    %r{([^-]+-[^-]+)$}
+                  ).flatten.first.to_s.gsub('.', '').gsub('-', '_'),
+                  :run_list => plat[:run_list],
+                  :attributes => plat[:attributes]
+              )
+            ]
+          end.flatten
+      )])
+    end
+
+    def generate_runlist(platform, suite)
+      r = platform_map[platform][:run_list]
+      kitchen_suite = kitchen[:suites].detect do |k_s|
+        k_s[:name] == suite
+      end
+      if(kitchen_suite && kitchen_suite[:run_list])
+        r |= kitchen_suite[:run_list]
+      end
+      r.uniq
+    end
+
+    def validate_platform!(plat)
+      unless(platform_map[plat])
+        ui.fatal "Requested platform does not exist: #{ui.color(plat, :red)}"
+        ui.info "  -> Available platforms: #{platform_map.keys.sort.join(', ')}"
+        exit EXIT_CODES[:kitchen_invalid_platform]
+      end
+    end
+  end
+end