kytoon 1.1.1 → 1.2.0

data/CHANGELOG

@@ -1,3 +1,10 @@
+* Wed Sep 19 2012 Dan Prince <dprince@redhat.com> - 1.2.0
+  - Add provider for OpenStack.
+  - Add bin/kytoon with commands corresponding to rake tasks.
+  - Move rake tasks under 'kytoon' namespace (previously used 'group').
+  - Updates to raise KytoonException instead of just raise.
+  - Rename gateway_ip task to just ip.
+
 * Thu Aug 23 2012 Dan Prince <dprince@redhat.com> - 1.1.1
   - Add 'libvirt_use_sudo' option to config file.
   - Move config parameter checks into specific providers.

data/Gemfile

@@ -10,6 +10,7 @@ group :development do
   gem "bundler", "~> 1.1.4"
   gem "builder", "~> 3.0.0"
   gem "jeweler", "~> 1.8.3"
+  gem "thor", "~> 0.14"
   gem "uuidtools", "~> 2.1.2"
   gem "mocha", "~> 0.12.1"
 end

data/Gemfile.lock

@@ -15,6 +15,7 @@ GEM
     rake (0.9.2.2)
     rdoc (3.12)
       json (~> 1.4)
+    thor (0.16.0)
     uuidtools (2.1.2)
 
 PLATFORMS
@@ -26,4 +27,5 @@ DEPENDENCIES
   jeweler (~> 1.8.3)
   mocha (~> 0.12.1)
   rdoc (~> 3.12)
+  thor (~> 0.14)
   uuidtools (~> 2.1.2)

data/README.md

@@ -0,0 +1,214 @@
+# Kytoon
+
+Create small virtual server groups
+
+## Description
+
+Provides both a CLI (bin/kytoon) and set of rake tasks to help automate the creation and configuration of virtual server groups. Kytoon provides the ability to create projects that can be used by team members and continuous integration systems to create similar (if not identical) groups of servers for development and/or testing. Configuration information is stored in JSON and YAML formats which can be easily parsed, edited, and version controlled.
+
+Inspired by and based on the Chef VPC Toolkit.
+
+## Features
+
+* Multiple providers (see below)
+* A CLI with matching set of rake tasks.
+* Create server groups with multiple nodes.
+* Automatically injects your ssh into each server group.
+* Configures hostnames (host files) on nodes within each group. Once you ssh
+  into the group use preconfigured hostnames to access all the nodes.
+
+## Provider support
+
+- Libvirt: manage instances on local machine w/ libvirt, virt-clone, and libguestfs
+- XenServer: manage instances on a remote XenServer box (via ssh)
+- OpenStack: create instances in the cloud.
+
+## Installation
+
+Gem install kytoon.
+
+    gem install kytoon
+
+Create a .kytoon.conf file in your $HOME directory.
+
+        # The default group type.
+        # Set to one of: openstack, libvirt, xenserver
+        group_type: openstack
+
+        # Openstack Settings
+        openstack_url: <auth_url>
+        openstack_username: <username>
+        openstack_password: <password>
+        openstack_network_name: public # Optional: defaults to public
+        openstack_keypair_name: < keyname > # Optional: file injection via personalities is the default
+        openstack_ip_type: 4 # IP type (4 or 6): defaults to 4
+        openstack_build_timeout: 480 # Server build timeout. Defaults to: 480
+        openstack_ping_timeout: 60 # Server build timeout. Defaults to: 60
+
+        # Libvirt settings
+        # Whether commands to create local group should use sudo
+        libvirt_use_sudo: False
+
+## Defining server groups
+
+Server group config files are used to define how Kytoon configures
+each server group.  These files typically live inside of project are
+provider specific. The config files control things like memory, hostname,
+flavor, etc. Each group should define identify one instance as the 'gateway' host which marks it as the primary access point for SSH access into the group.
+
+By default Kytoon looks for config/server_group.json in the current directory.
+You can override this with Rake using GROUP_CONFIG or bin/kytoon using --group-config.
+
+Below are example server_group.json config files for each provider.
+
+For Openstack:
+
+```bash
+
+	cat > config/server_group.json <<-"EOF_CAT"
+	{
+	"name": "Fedora",
+	"servers": [
+		{
+		"hostname": "nova1",
+		"image_ref": "1234",
+		"flavor_ref": "5678",
+		"gateway": "true"
+		},
+		{
+		"hostname": "compute1",
+		"image_ref": "1234",
+		"flavor_ref": "5678"
+		}
+	]
+	}
+	EOF_CAT
+```
+
+For Libvirt (uses libvirt DHCP server for instance IP configuration):
+
+```bash
+
+	cat > config/server_group.json <<-"EOF_CAT"
+	{
+	"name": "Fedora",
+	"servers": [
+		{
+		"hostname": "nova1",
+		"memory": "1",
+		"gateway": "true",
+		"original_xml": "/home/dprince/f17.xml",
+		"create_cow": "true"
+		}
+	]
+	}
+	EOF_CAT
+```
+
+For XenServer (uses Openstack Guest agent for instance configuration):
+```bash
+
+        cat > config/server_group.json <<-"EOF_CAT"
+	{
+	"name": "Fedora",
+	"netmask": "255.255.255.0",
+	"gateway": "192.168.0.1",
+	"broadcast": "192.168.0.127",
+	"dns_nameserver": "8.8.8.8",
+	"network_type": "static",
+	"public_ip_bridge": "xenbr0",
+	"bridge": "xenbr1",
+	"servers": [
+		{
+		"hostname": "login",
+		"image_path": "/images/fedora-agent2.xva",
+		"ip_address": "192.168.0.2",
+		"mac": "e2:6d:71:67:7e:66"
+		},
+		{
+		"hostname": "nova1",
+		"image_path": "/images/fedora-agent2.xva",
+		"ip_address": "192.168.0.3",
+		"mac": "e2:ad:a1:a7:ae:67"
+		}
+	    ]
+	}
+	EOF_CAT
+```
+
+## Command line
+
+The following options are supported on the command line:
+
+	kytoon create       # Create a new server group.
+	kytoon delete       # Delete a server group.
+	kytoon help [TASK]  # Describe available tasks or one specific task
+	kytoon ip           # Print the IP address of the gateway server
+	kytoon list         # List existing server groups.
+	kytoon show         # Print information for a server group.
+	kytoon ssh          # SSH into a group.
+
+## Rakefile configuration
+
+To include Kytoon rake tasks in your own project add the following to your
+Rakefile:
+
+	KYTOON_PROJECT = "#{File.dirname(__FILE__)}" unless defined?(KYTOON_PROJECT)
+	require 'rubygems'
+	require 'kytoon'
+	include Kytoon
+	Dir[File.join("#{Kytoon::Version::KYTOON_ROOT}/rake", '*.rake')].each do  |rakefile|
+		import(rakefile)
+	end
+
+## Rake Tasks
+
+Example commands:
+
+* Create a new server group.
+
+	$ rake kytoon:create
+
+* List your currently running server groups.
+
+	$ rake kytoon:list
+
+* SSH into the current (most recently created) server group
+
+	$ rake kytoon:ssh
+
+* SSH into a server group with an ID of 3
+
+	$ rake kytoon:ssh GROUP_ID=3
+
+* Delete the server group with an ID of 3
+
+	$ rake kytoon:delete GROUP_ID=3
+
+
+## Bash Automation Script
+
+The following is an example bash script to spin up a group and run commands via SSH.
+
+```bash
+        #!/bin/bash
+        # override the group type specified in .kytoon.conf
+        export GROUP_TYPE=libvirt
+
+        trap "rake kytoon:delete" INT TERM EXIT # cleanup the group on exit
+
+        # create a server group (uses config/server_group.json)
+        rake kytoon:create
+
+        # create a server group with alternate json file
+        rake kytoon:create GROUP_CONFIG=config/my_group.json
+
+        # Run some scripts on the login server
+        rake kytoon:ssh bash <<-EOF_BASH
+                echo 'It works!'
+        EOF_BASH
+```
+
+## Copyright
+
+Copyright (c) 2012 Red Hat Inc. See LICENSE.txt for further details.

data/Rakefile

@@ -18,17 +18,19 @@ Rake::Task['test'].comment = "Unit"
 
 begin
   require 'jeweler'
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = "kytoon"
-    gemspec.summary = "Create & configure ephemeral virtual private clouds."
-    gemspec.description = "A set of Rake tasks that provide a framework to help automate the creation and configuration of VPC server groups."
-    gemspec.email = "dprince@redhat.com"
-    gemspec.homepage = "http://github.com/dprince/kytoon"
-    gemspec.authors = ["Dan Prince"]
-    gemspec.add_dependency 'rake'
-    gemspec.add_dependency 'builder'
-    gemspec.add_dependency 'json'
-    gemspec.add_dependency 'uuidtools'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "kytoon"
+    gem.summary = "Create & configure ephemeral virtual private clouds."
+    gem.description = "A set of Rake tasks that provide a framework to help automate the creation and configuration of VPC server groups."
+    gem.email = "dprince@redhat.com"
+    gem.homepage = "http://github.com/dprince/kytoon"
+    gem.authors = ["Dan Prince"]
+    gem.add_dependency 'rake'
+    gem.add_dependency 'builder'
+    gem.add_dependency 'json'
+    gem.add_dependency 'openstack-compute'
+    gem.add_dependency 'thor'
+    gem.add_dependency 'uuidtools'
   end
 rescue LoadError
   puts "Jeweler not available. Install it with: sudo gem install jeweler"

data/VERSION

@@ -1 +1 @@
-1.1.1
+1.2.0

data/bin/kytoon

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "kytoon"
+
+KYTOON_PROJECT = ENV['PWD']
+
+Kytoon::ThorTasks.start

data/config/server_group_openstack.json

@@ -0,0 +1,12 @@
+{
+    "name": "Fedora",
+    "servers": [
+        {
+            "hostname": "nova1",
+            "image_ref": "1234",
+            "flavor_ref": "5678",
+            "gateway": "true",
+            "keypair": "test"
+        }
+    ]
+}

data/lib/kytoon.rb

@@ -1,5 +1,7 @@
 require 'kytoon/util'
 require 'kytoon/ssh_util'
+require 'kytoon/thor_tasks'
+require 'kytoon/exception'
 require 'kytoon/version'
 require 'kytoon/xml_util'
 require 'kytoon/server_group'

data/lib/kytoon/exception.rb

@@ -0,0 +1,8 @@
+class KytoonException < StandardError
+end
+
+class ConfigException < KytoonException
+end
+
+class NoServerGroupExists < KytoonException
+end

data/lib/kytoon/providers/libvirt/server_group.rb

@@ -170,7 +170,7 @@ fi
       }, server['ip_address']) do |ok, out|
         if not ok
           puts out
-          raise "Failed to copy host file to instance #{server['hostname']}."
+          raise KytoonException, "Failed to copy host file to instance #{server['hostname']}."
         end
       end
     end
@@ -182,12 +182,12 @@ fi
     id = options[:id]
     if id.nil? then
       group=ServerGroup.most_recent
-      raise "No server group files exist." if group.nil?
+      raise NoServerGroupExists, "No server group files exist." if group.nil?
       id=group.id
     end
 
     out_file=File.join(@@data_dir, "#{id}.json")
-    raise "No server group files exist." if not File.exists?(out_file)
+    raise NoServerGroupExists, "No server group files exist." if not File.exists?(out_file)
     ServerGroup.from_json(IO.read(out_file))
   end
 
@@ -223,7 +223,7 @@ fi
     REXML::XPath.each(dom, "//disk[1]/source") do |source_xml|
       return source_xml.attributes['file']
     end
-    raise "Unable to find disk path for instance."
+    raise KytoonException, "Unable to find disk path for instance."
   end
 
   def self.create_instance(group_id, inst_name, memory_gigs, original, original_xml, disk_path, create_cow, ssh_public_key, sudo)
@@ -288,7 +288,7 @@ fi
     retval=$?
     if not retval.success? 
       puts out
-      raise "Failed to create instance #{inst_name}."
+      raise KytoonException, "Failed to create instance #{inst_name}."
     end
 
     # lookup server IP here... 
@@ -298,7 +298,7 @@ fi
     REXML::XPath.each(dom, "//interface/mac") do |interface_xml|
       mac_addr = interface_xml.attributes['address']
     end
-    raise "Failed to lookup mac address for #{inst_name}" if mac_addr.nil?
+    raise KytoonException, "Failed to lookup mac address for #{inst_name}" if mac_addr.nil?
 
     instance_ip = %x{grep -i #{mac_addr} /var/lib/libvirt/dnsmasq/default.leases | cut -d " " -f 3}.chomp
     count = 0
@@ -307,7 +307,7 @@ fi
       sleep 1
       count += 1
       if count >= 60 then
-          raise "Failed to lookup ip address for #{inst_name}"
+          raise KytoonException, "Failed to lookup ip address for #{inst_name}"
       end
     end
     return instance_ip
@@ -335,7 +335,7 @@ fi
     retval=$?
     if not retval.success? 
       puts out
-      raise "Failed to cleanup instances."
+      raise KytoonException, "Failed to cleanup instances."
     end
   end
 

data/lib/kytoon/providers/openstack.rb

@@ -0,0 +1,5 @@
+require 'kytoon/providers/openstack/server_group'
+
+Kytoon::Util.check_config_param('openstack_url')
+Kytoon::Util.check_config_param('openstack_username')
+Kytoon::Util.check_config_param('openstack_password')

data/lib/kytoon/providers/openstack/server_group.rb

@@ -0,0 +1,317 @@
+require 'json'
+require 'kytoon/util'
+require 'openstack/compute'
+
+module Kytoon
+
+module Providers
+
+module Openstack
+
+# Openstack server group provider.
+class ServerGroup
+
+  @@connection=nil
+  @@data_dir=File.join(KYTOON_PROJECT, "tmp", "openstack")
+
+  def self.data_dir
+    @@data_dir
+  end
+
+  def self.data_dir=(dir)
+    @@data_dir=dir
+  end
+
+  CONFIG_FILE = KYTOON_PROJECT + File::SEPARATOR + "config" + File::SEPARATOR + "server_group.json"
+
+  attr_accessor :id
+  attr_accessor :name
+
+  def initialize(options={})
+    @id = options[:id] || Time.now.to_i
+    @name = options[:name]
+    @servers=[]
+    end
+
+  def server(name)
+    @servers.select {|s| s['hostname'] == name}[0] if @servers.size > 0
+  end
+
+  def servers
+    @servers
+  end
+
+  def gateway_ip
+    @servers.select {|s| s['gateway'] == 'true' }[0]['ip_address'] if @servers.size > 0
+  end
+
+  # generate a Server Group XML from server_group.json
+  def self.from_json(json)
+
+    json_hash=JSON.parse(json)
+
+    sg=ServerGroup.new(
+      :id => json_hash["id"],
+      :name => json_hash["name"]
+    )
+    json_hash["servers"].each do |server_hash|
+
+      sg.servers << {
+        'id' => server_hash['id'],
+        'hostname' => server_hash['hostname'],
+        'image_ref' => server_hash['image_ref'],
+        'flavor_ref' => server_hash['flavor_ref'],
+        'keypair_name' => server_hash['keypair_name'],
+        'gateway' => server_hash['gateway'] || "false",
+        'ip_address' => server_hash['ip_address']
+      }
+    end
+    return sg
+  end
+
+  def pretty_print
+
+    puts "Group ID: #{@id}"
+    puts "name: #{@name}"
+    puts "gateway IP: #{self.gateway_ip}"
+    puts "Servers:"
+    servers.each do |server|
+      puts "\tname: #{server['hostname']}"
+      puts "\t--"
+    end
+
+  end
+
+  def server_names
+
+    names=[]  
+
+    servers.each do |server|
+      if block_given? then
+        yield server['hostname']
+      else
+        names << server['hostname']
+      end  
+    end
+
+    names
+    
+  end
+
+  def cache_to_disk
+
+    sg_hash = {
+        'id' => @id,
+        'name' => @name,
+        'servers' => []
+    }
+    @servers.each do |server|
+        sg_hash['servers'] << {'id' => server['id'], 'hostname' => server['hostname'], 'image_ref' => server['image_ref'], 'gateway' => server['gateway'], 'flavor_ref' => server['flavor_ref'], 'ip_address' => server['ip_address']}
+    end
+
+    FileUtils.mkdir_p(@@data_dir)
+    File.open(File.join(@@data_dir, "#{@id}.json"), 'w') do |f|
+      f.chmod(0600)
+      f.write(sg_hash.to_json)
+    end
+  end
+
+  def delete
+    servers.each do |server|
+      ServerGroup.destroy_instance(server['id'])
+    end
+    out_file=File.join(@@data_dir, "#{@id}.json")
+    File.delete(out_file) if File.exists?(out_file)
+  end
+
+
+  def self.create(sg)
+
+    hosts_file_data = "127.0.0.1\tlocalhost localhost.localdomain\n"
+
+    build_timeout = (Util.load_configs['openstack_build_timeout'] || 60).to_i
+
+    sg.servers.each do |server|
+      server_id = create_instance(sg.id, server['hostname'], server['image_ref'], server['flavor_ref'], server['keypair_name']).id
+
+      server['id'] = server_id
+      sg.cache_to_disk
+    end
+
+    begin
+      Timeout::timeout(build_timeout) do
+        ips = get_server_ips
+        sg.servers.each do |server|
+          server_ip = ips[server['id']]
+          server['ip_address'] = server_ip
+          sg.cache_to_disk
+          hosts_file_data += "#{server_ip}\t#{server['hostname']}\n"
+        end
+      end
+    rescue Timeout::Error => te
+      raise KytoonException, "Timeout building server group."
+    end
+
+
+    puts "Copying hosts files..."
+    #now that we have IP info copy hosts files into the servers
+    sg.servers.each do |server|
+      ping_test(server['ip_address'])
+      Kytoon::Util.remote_exec(%{
+cat > /etc/hosts <<-EOF_CAT
+#{hosts_file_data}
+EOF_CAT
+hostname "#{server['hostname']}"
+if [ -f /etc/sysconfig/network ]; then
+  sed -e "s|^HOSTNAME.*|HOSTNAME=#{server['hostname']}|" -i /etc/sysconfig/network
+fi
+      }, server['ip_address']) do |ok, out|
+        if not ok
+          puts out
+          raise KytoonException, "Failed to copy host file to instance #{server['hostname']}."
+        end
+      end
+    end
+
+    sg
+
+  end
+
+  def self.get(options={})
+    id = options[:id]
+    if id.nil? then
+      group=ServerGroup.most_recent
+      raise NoServerGroupExists, "No server group files exist." if group.nil?
+      id=group.id
+    end
+
+    out_file=File.join(@@data_dir, "#{id}.json")
+    raise NoServerGroupExists, "No server group files exist." if not File.exists?(out_file)
+    ServerGroup.from_json(IO.read(out_file))
+  end
+
+  def self.index(options={})
+
+    server_groups=[]
+    Dir[File.join(ServerGroup.data_dir, '*.json')].each do  |file|
+      server_groups << ServerGroup.from_json(IO.read(file))
+    end
+    server_groups
+
+  end
+
+  def self.most_recent
+    server_groups=[]
+    Dir[File.join(@@data_dir, "*.json")].each do  |file|
+      server_groups << ServerGroup.from_json(IO.read(file))
+    end
+    if server_groups.size > 0 then
+      server_groups.sort { |a,b| b.id <=> a.id }[0]
+    else
+      nil
+    end
+  end
+
+  def self.init_connection
+    configs = Util.load_configs
+    if @@connection.nil? then
+      @@connection = OpenStack::Compute::Connection.new(
+        :username => configs['openstack_username'].to_s,
+        :api_key => configs['openstack_password'].to_s,
+        :auth_url => configs['openstack_url'],
+        :retry_auth => false)
+    else
+      @@connection
+    end
+  end
+
+  def self.create_instance(group_id, hostname, image_ref, flavor_ref, keypair_name)
+
+    ssh_public_key = Util.public_key_path
+    configs = Util.load_configs
+
+    conn = self.init_connection
+
+    options = {
+      :name => "#{group_id}_#{hostname}",
+      :imageRef => image_ref,
+      :flavorRef => flavor_ref,
+      :personality => {ssh_public_key => "/root/.ssh/authorized_keys"},
+      :is_debug => true}
+
+    keypair_name = configs['openstack_keypair_name'] if keypair_name.nil?
+    if not keypair_name.nil? and not keypair_name.empty? then
+      options.store(:key_name, keypair_name)
+    end
+
+    conn.create_server(options)
+
+  end
+
+  def self.default_ip_type()
+    ip_type = Util.load_configs['openstack_ip_type'] || 4
+    ip_type.to_i
+  end
+
+  def self.get_server_ips()
+
+    ips = {}
+    configs = Util.load_configs
+
+    network_name = configs['openstack_network_name'] || 'public'
+
+    conn = self.init_connection
+    all_active = false
+    until all_active do
+      all_active = true
+      conn.servers.each do |server|
+        server = conn.server(server[:id])
+        if server.status == 'ACTIVE' and ips[server.id].nil? then
+          addresses = server.addresses[network_name.to_sym].select {|a| a.version == self.default_ip_type}
+          ips[server.id] = addresses[0].address
+        else
+          all_active = false
+        end
+      end
+    end
+
+    ips
+
+  end
+
+  def self.ping_test(ip_addr)
+
+    ping_timeout = (Util.load_configs['openstack_ping_timeout'] || 60).to_i
+
+    begin
+      ping = self.default_ip_type == 6 ? 'ping6' : 'ping'
+      ping_command = "#{ping} -c 1 #{ip_addr} > /dev/null 2>&1"
+      Timeout::timeout(ping_timeout) do
+        while(1) do
+          return true if system(ping_command)
+        end
+      end
+    rescue Timeout::Error => te
+      raise KytoonException, "Timeout pinging server: #{ping_command}"
+    end
+
+    return false
+
+  end
+
+  def self.destroy_instance(uuid)
+    begin
+      conn = self.init_connection
+      conn.server(uuid).delete!
+    rescue Exception => e
+      puts "Error deleting server: #{e.message}"
+    end
+  end
+
+end
+
+end
+
+end
+
+end