rivet 1.0.0

data/lib/rivet/bootstrap.rb

@@ -0,0 +1,62 @@
+module Rivet
+  class Bootstrap
+    TEMPLATE_SUB_DIR = "bootstrap"
+
+    attr_reader :gems, :run_list, :template, :environment
+    attr_reader :template_path, :chef_command, :chef_organization
+
+    def initialize(bootstrap_definition = Hash.new)
+      ivars = [
+        'gems','run_list','template','environment',
+        'config_dir','chef_organization']
+
+      ivars.each do |i|
+        if bootstrap_definition.has_key?(i)
+          instance_variable_set("@#{i}",bootstrap_definition[i])
+        end
+      end unless bootstrap_definition.nil?
+
+      @config_dir ||= "."
+      @template   ||= "default.erb"
+
+      set_calculated_attrs
+    end
+
+    def user_data
+      @user_data ||= generate_user_data
+    end
+
+    protected
+
+    def set_calculated_attrs
+      @template_path  = File.join(@config_dir,TEMPLATE_SUB_DIR)
+      @chef_command   = "/usr/bin/chef-client -j /etc/chef/first-boot.json -L /root/first_run.log -E #{@environment}"
+      @secret_file    = File.join(@config_dir,"encrypted_data_bag_secret_#{@environment}")
+      @validation_key = File.new(File.join(@config_dir,"#{@chef_organization}-validator.pem")).read
+    end
+
+    def generate_user_data
+      config_content = "log_level :info\n"
+      config_content << "log_location STDOUT\n"
+      config_content << "environment #{environment}\n"
+      config_content << "chef_server_url  'https://api.opscode.com/organizations/#{chef_organization}'\n"
+      config_content << "validation_client_name '#{chef_organization}-validator'\n"
+
+      install_gems = String.new
+
+      gems.each do |gem|
+        if gem.size > 1
+          install_gems << "gem install #{gem[0]} -v #{gem[1]} --no-rdoc --no-ri\n"
+        else
+          install_gems << "gem install #{gem[0]} --no-rdoc --no-ri\n"
+        end
+      end unless gems.nil?
+
+      first_boot = { :run_list => @run_list.join(",") }.to_json unless @run_list.nil?
+
+      template = ERB.new File.new(File.join(@template_path,@template)).read
+      template.result(binding)
+    end
+
+  end
+end

data/lib/rivet/client.rb

@@ -0,0 +1,29 @@
+module Rivet
+  class Client
+    def initialize
+    end
+
+    def run(options)
+      AwsUtils.set_aws_credentials(options[:profile])
+      Rivet::Log.level(options[:log_level])
+      Rivet::Utils.ensure_minimum_setup
+
+      group_def = Rivet::Utils.get_definition(options[:group])
+
+      Rivet::Utils.die "The #{options[:group]} definition doesn't exist" unless group_def
+
+      Rivet::Log.info("Checking #{options[:group]} autoscaling definition")
+      autoscale_def = Rivet::Autoscale.new(options[:group],group_def)
+      autoscale_def.show_differences
+
+      if options[:sync]
+        autoscale_def.sync
+      else
+        Rivet::Log.info("use the -s [--sync] flag to sync changes")
+      end
+
+    end
+
+  end
+end
+

data/lib/rivet/deep_merge.rb

@@ -0,0 +1,27 @@
+class Hash
+  # Returns a new hash with +self+ and +other_hash+ merged recursively.
+  #
+  #   h1 = { x: { y: [4,5,6] }, z: [7,8,9] }
+  #   h2 = { x: { y: [7,8,9] }, z: 'xyz' }
+  #
+  #   h1.deep_merge(h2) #=> {x: {y: [7, 8, 9]}, z: "xyz"}
+  #   h2.deep_merge(h1) #=> {x: {y: [4, 5, 6]}, z: [7, 8, 9]}
+  #   h1.deep_merge(h2) { |key, old, new| Array.wrap(old) + Array.wrap(new) }
+  #   #=> {:x=>{:y=>[4, 5, 6, 7, 8, 9]}, :z=>[7, 8, 9, "xyz"]}
+  def deep_merge(other_hash, &block)
+    dup.deep_merge!(other_hash, &block)
+  end
+
+  # Same as +deep_merge+, but modifies +self+.
+  def deep_merge!(other_hash, &block)
+    other_hash.each_pair do |k,v|
+      tv = self[k]
+      if tv.is_a?(Hash) && v.is_a?(Hash)
+        self[k] = tv.deep_merge(v, &block)
+      else
+        self[k] = block && tv ? block.call(k, tv, v) : v
+      end
+    end
+    self
+  end
+end

data/lib/rivet/launch_config.rb

@@ -0,0 +1,90 @@
+module Rivet
+  class LaunchConfig
+
+    LC_ATTRIBUTES = ['key_name','image_id','instance_type','security_groups','bootstrap']
+
+    LC_ATTRIBUTES.each do |a|
+      attr_reader a.to_sym
+    end
+
+    attr_reader :id_prefix
+
+    def initialize(spec,id_prefix="rivet_")
+      @id_prefix = id_prefix
+
+      LC_ATTRIBUTES.each do |a|
+
+        if respond_to? "normalize_#{a}".to_sym
+          spec[a] = self.send("normalize_#{a.to_sym}",spec[a])
+        end
+
+        instance_variable_set("@#{a}",spec[a])
+      end
+    end
+
+    def user_data
+      @user_data ||= Bootstrap.new(bootstrap).user_data
+    end
+
+    def identity
+      @identity ||= generate_identity
+    end
+
+    def save
+      AwsUtils.verify_security_groups(security_groups)
+
+      lc_collection = AWS::AutoScaling.new().launch_configurations
+
+      if lc_collection[identity].exists?
+        Rivet::Log.info("Launch configuration #{identity} already exists in AWS")
+      else
+        options = { :key_pair => key_name, :security_groups => security_groups, :user_data => user_data}
+        Rivet::Log.info("Saving launch configuration #{identity} to AWS")
+        Rivet::Log.debug("Launch Config options:\n #{options.inspect}")
+        lc_collection.create(identity,image_id,instance_type, options)
+      end
+    end
+
+    protected
+
+    def build_identity_string
+      identity = LC_ATTRIBUTES.inject(String.new) do |accum,attribute|
+        if attribute != 'bootstrap'
+          attr_value = self.send(attribute.to_sym) ? self.send(attribute.to_sym) : "\0"
+          attr_value = attr_value.join("\t") if attr_value.respond_to?(:join)
+          accum << attribute.to_s
+          accum << Base64.encode64(attr_value)
+        else
+          accum << attribute.to_s
+          accum << Base64.encode64(user_data ? user_data : "\0")
+        end
+        accum
+      end
+      identity
+    end
+
+    def generate_identity
+      @id_prefix + Digest::SHA1.hexdigest(build_identity_string)
+    end
+
+    def old_generate_identity
+      identity = LC_ATTRIBUTES.inject({}) do |ident_hash,attribute|
+        if attribute != 'bootstrap'
+          Rivet::Log.debug("Adding #{attribute} : #{self.send(attribute.to_sym)} to identity hash for LaunchConfig")
+          ident_hash[attribute] = self.send(attribute.to_sym)
+        else
+          Rivet::Log.debug("Adding user_data to identity hash for LaunchConfig:\n#{user_data} ")
+          ident_hash[attribute] = user_data
+        end
+        ident_hash
+      end
+      @id_prefix + Digest::SHA1.hexdigest(Marshal::dump(identity))
+    end
+
+    def normalize_security_groups(groups)
+      groups.sort
+    end
+
+  end
+end
+

data/lib/rivet/logger.rb

@@ -0,0 +1,52 @@
+module Rivet
+
+  module Log
+
+    def self.write(level,message)
+      @@log ||= SimpleLogger.instance
+      @@log.send(level.to_sym) { message }
+    end
+
+    def self.info(message)
+      write('info',message)
+    end
+
+    def self.debug(message)
+      write('debug',message)
+    end
+
+    def self.fatal(message)
+      write('fatal',message)
+    end
+
+    def self.warn(message)
+      write('warn',message)
+    end
+
+    def self.level(level)
+      @@log ||= SimpleLogger.instance
+      @@log.level = level
+    end
+
+    class SimpleLogger< Logger
+      include Singleton
+
+      def initialize
+        @dev = Logger::LogDevice.new(STDOUT)
+        super @dev
+        @progname = "Rivet"
+        @formatter = proc do |sev,datetime,name,msg|
+          "[#{name}] [#{datetime}] [#{sev}]: #{msg}\n"
+        end
+        @datetime_format
+      end
+
+      def close
+        @dev.close
+      end
+    end
+
+  end
+end
+
+

data/lib/rivet/utils.rb

@@ -0,0 +1,68 @@
+module Rivet
+  module Utils
+    AUTOSCALE_DIR = "autoscale"
+
+    def self.die(level = 'fatal',message)
+      Rivet::Log.write(level,message)
+      exit
+    end
+
+    def self.ensure_minimum_setup
+      if Dir.exists?(AUTOSCALE_DIR)
+        true
+      else
+        Rivet::Log.info("Creating #{AUTOSCALE_DIR}")
+        Dir.mkdir(AUTOSCALE_DIR)
+      end
+    end
+
+    # This returns the merged definition given a group
+
+    def self.get_definition(group)
+      defaults = consume_defaults
+      group_def = load_definition(group)
+      if defaults && group_def
+        group_def = defaults.deep_merge(group_def)
+      end
+    group_def ? group_def : false
+    end
+
+    # Gobbles up the defaults file from YML, returns the hash or false if empty
+
+    def self.consume_defaults(autoscale_dir = AUTOSCALE_DIR)
+      defaults_file = File.join(autoscale_dir,"defaults.yml")
+      if File.exists?(defaults_file)
+        parsed = begin
+          Rivet::Log.debug("Consuming defaults from #{defaults_file}")
+          YAML.load(File.open(defaults_file))
+        rescue ArgumentError => e
+          Rivet::Log.fatal("Could not parse YAML from #{defaults_file}: #{e.message}")
+        end
+        parsed
+      else
+        false
+      end
+    end
+
+    # This loads the given definition from it's YML file, returns the hash or
+    # false if empty
+
+    def self.load_definition(name)
+      definition_dir = File.join(AUTOSCALE_DIR,name)
+      conf_file      = File.join(definition_dir,"conf.yml")
+      if Dir.exists?(definition_dir) && File.exists?(conf_file)
+        Rivet::Log.debug("Loading definition for #{name} from #{conf_file}")
+        parsed = begin
+          YAML.load(File.open(conf_file))
+        rescue
+          Rivet::Log.fatal("Could not parse YAML from #{conf_file}: #{e.message}")
+        end
+        parsed ? parsed : { }
+      else
+        false
+      end
+    end
+
+  end
+end
+

data/lib/rivet/version.rb

@@ -0,0 +1,4 @@
+module Rivet
+  VERSION = "1.0.0"
+end
+

data/lib/rivet.rb

@@ -0,0 +1,18 @@
+require 'aws-sdk'
+require 'base64'
+require 'digest/sha1'
+require 'erb'
+require 'json'
+require 'logger'
+require 'optparse'
+require 'singleton'
+require 'yaml'
+
+require_relative 'rivet/deep_merge'
+require_relative 'rivet/logger'
+require_relative 'rivet/utils'
+require_relative 'rivet/aws_utils'
+require_relative 'rivet/launch_config'
+require_relative 'rivet/autoscale'
+require_relative 'rivet/bootstrap'
+require_relative 'rivet/client'

data/rivet.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path('../lib/',__FILE__)
+$:.unshift lib unless $:.include?(lib)
+require 'rivet/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = 'rivet'
+  spec.version     = Rivet::VERSION
+  spec.licenses    = ['Apache2']
+  spec.authors     = ['Brian Bianco']
+  spec.email       = ['brian.bianco@gmail.com']
+  spec.homepage    = 'http://www.github.com/brianbianco/rivet'
+  spec.summary     = %q{A tool for managing autoscaling groups}
+  spec.description = %q{Rivet allows you to define autoscaling groups and launch configurations as YAML and can SYNC that to AWS}
+
+  spec.required_ruby_version     = '>= 1.9.1'
+  spec.required_rubygems_version = '>= 1.3.6'
+
+  spec.files = `git ls-files`.split($/)
+  spec.test_files = spec.files.grep(%r{^spec/})
+
+  spec.executables  = %w(rivet)
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-sdk", "~> 1.24.0"
+  spec.add_development_dependency "rake", ">= 10.1.0"
+  spec.add_development_dependency "rspec", "~> 2.14.1"
+end
+
+

data/spec/rivet_bootstrap_spec.rb

@@ -0,0 +1,69 @@
+require_relative './rivet_spec_setup'
+
+include SpecHelpers
+
+describe 'rivet bootstrap' do
+  let (:bootstrap) { Rivet::Bootstrap.new(SpecHelpers::AUTOSCALE_DEF['bootstrap']) }
+  let (:bootstrap_def) { SpecHelpers::AUTOSCALE_DEF['bootstrap'] }
+
+  tempdir_context 'with all necessary files in place' do
+    before do
+
+
+      validator_file = File.join(
+        bootstrap_def['config_dir'],
+        "#{bootstrap_def['environment']}-validator.pem")
+
+      template_dir = File.join(
+        bootstrap_def['config_dir'],
+        Rivet::Bootstrap::TEMPLATE_SUB_DIR)
+
+      template_file = File.join(template_dir,bootstrap_def['template'])
+
+      FileUtils.mkdir_p(bootstrap_def['config_dir'])
+      FileUtils.mkdir_p(template_dir)
+      File.open(template_file,'w') { |f| f.write(SpecHelpers::BOOTSTRAP_TEMPLATE) }
+      FileUtils.touch(validator_file)
+    end
+
+    describe "#user_data" do
+      it 'returns a string that contains the chef organization' do
+        org = bootstrap_def['organization']
+        bootstrap.user_data.should =~ /chef_server_url\s*.*#{org}.*/
+      end
+
+      it 'returns a string that contains the environment' do
+        env = bootstrap_def['env']
+        bootstrap.user_data.should =~ /environment\s*.*#{env}.*/
+      end
+
+      it 'returns a string that contains the run_list as json' do
+        run_list = { :run_list => bootstrap_def['run_list'].join(",") }.to_json
+        bootstrap.user_data.should =~ /#{run_list}/
+      end
+
+      it 'returns a string that contains each gem to install' do
+        bootstrap_def['gems'].each do |g|
+          if g.size > 1
+            gem_regexp = /gem\s*install\s*#{g[0]}.*#{g[1]}/
+          else
+            gem_regexp = /gem\s*install\s*#{g[0]}/
+          end
+          bootstrap.user_data.should =~ gem_regexp
+        end
+      end
+
+    end
+
+  end
+
+end
+
+
+
+
+
+
+
+
+

data/spec/rivet_launch_config_spec.rb

@@ -0,0 +1,37 @@
+require_relative './rivet_spec_setup'
+
+include SpecHelpers
+
+describe "rivet launch config" do
+  let (:launch_config) { Rivet::LaunchConfig.new(SpecHelpers::AUTOSCALE_DEF) }
+
+  context "with a valid autoscale definition" do
+    before do
+      user_data_mock = double('user_data_mock')
+      user_data_mock.stub(:user_data).and_return("unit_test_user_data")
+      Rivet::Bootstrap.stub(:new).and_return(user_data_mock)
+    end
+
+    describe "#build_identity_string" do
+      it "should return a valid identity_string" do
+        launch_config.send(:build_identity_string).should == SpecHelpers::AUTOSCALE_IDENTITY_STRING
+      end
+    end
+
+    describe "#identity" do
+      it "should return a deterministic identity" do
+        launch_config.identity.should == "rivet_#{Digest::SHA1.hexdigest(SpecHelpers::AUTOSCALE_IDENTITY_STRING)}"
+      end
+    end
+
+    describe "#normalize_security_groups" do
+      it "returns a sorted array of groups" do
+        unsorted_groups = ['group3','group1','group2']
+        sorted_groups = unsorted_groups.sort
+        returned_groups = launch_config.send(:normalize_security_groups,unsorted_groups)  
+        returned_groups.should == sorted_groups
+      end
+    end
+  end
+
+end

data/spec/rivet_spec_setup.rb

@@ -0,0 +1,62 @@
+require 'rspec'
+require 'fileutils'
+require 'tempfile'
+require 'pathname'
+require 'base64'
+require_relative '../lib/rivet'
+
+Rivet::Log.level(Logger::FATAL)
+
+
+module SpecHelpers
+
+
+   BOOTSTRAP_TEMPLATE = '<%= install_gems %>'\
+                        '<%= config_content %>'\
+                        '<%= first_boot %>'\
+                        "\n"\
+                        '<%= chef_command %>'
+
+
+  AUTOSCALE_DEF = {
+    'min_size' => 1,
+    'max_size' => 3,
+    'region'   => 'us-west-2',
+    'availability_zones' => ['a','b','c'],
+    'key_name' => 'UnitTests',
+    'instance_type' => 'm1.large',
+    'security_groups' => ['unit_tests1','unit_tests2'],
+    'image_id' => 'ami-12345678',
+    'bootstrap' => {
+      'chef_organization' => 'unit_tests',
+      'template' => 'default.erb',
+      'config_dir' => 'unit_tests',
+      'environment' => 'unit_tests',
+      'gems' => [ ['gem1','0.0.1'],['gem2','0.0.2'] ],
+      'run_list' => ['unit_tests']
+    }
+  }
+
+  AUTOSCALE_IDENTITY_STRING = "key_name#{Base64.encode64(AUTOSCALE_DEF['key_name'])}"\
+                              "image_id#{Base64.encode64(AUTOSCALE_DEF['image_id'])}"\
+                              "instance_type#{Base64.encode64(AUTOSCALE_DEF['instance_type'])}"\
+                              "security_groups#{Base64.encode64(AUTOSCALE_DEF['security_groups'].join("\t"))}"\
+                              "bootstrap#{Base64.encode64('unit_test_user_data')}"\
+
+  def tempdir_context(name, &block)
+    context name do
+      before do
+        @origin_dir = Dir.pwd
+        @temp_dir = ::Pathname.new(::File.expand_path(::Dir.mktmpdir))
+        Dir.chdir @temp_dir
+      end
+
+      after do
+        Dir.chdir @origin_dir
+        FileUtils.remove_entry(@temp_dir)
+      end
+
+      instance_eval &block
+    end
+  end
+end

data/spec/rivet_util_spec.rb

@@ -0,0 +1,123 @@
+require_relative './rivet_spec_setup'
+
+include SpecHelpers
+
+definition_name = "unit_test"
+definition_dir  = File.join(Rivet::Utils::AUTOSCALE_DIR,definition_name)
+launch_config_params = ['ssh_key','instance_size','security_groups','ami','bootstrap']
+
+defaults_hash = {
+  'min_size' => 0,
+  'max_size' => 0,
+  'region' => 'us-west-2',
+  'zones' => ['a','b','c'],
+  'key_name' => 'unit_tests',
+  'instance_type' => 'm1.large',
+  'security_groups' => ['unit_tests'],
+  'image_id' => 'ami-unit_tests',
+  'bootstrap' => {
+    'run_list' => ['role[unit_tests]']
+  }
+}
+
+unit_test_definition_hash = {
+  'min_size' => 1,
+  'max_size' => 5,
+  'bootstrap' => {
+    'run_list' => ['role[merging_test']
+  }
+}
+
+describe "rivet utils" do
+  tempdir_context "without an autoscaling directory" do
+    describe "ensure_minimum_setup" do
+      it "creates the autoscale directory if it doesn't exist" do
+        Rivet::Utils.ensure_minimum_setup
+        Dir.exists?(Rivet::Utils::AUTOSCALE_DIR).should be_true
+      end
+    end
+  end
+
+  tempdir_context "with an autoscaling directory" do
+    before do
+      FileUtils.mkdir_p(Rivet::Utils::AUTOSCALE_DIR)
+    end
+
+    describe "ensure_minimum_setup" do
+      it "should return true" do
+        Rivet::Utils.ensure_minimum_setup.should be_true
+      end
+    end
+
+    describe "consume_defaults" do
+      it "should return false" do
+        Rivet::Utils.consume_defaults.should be_false
+      end
+    end
+
+    describe "load_definition" do
+      it "should return false" do
+        Rivet::Utils.load_definition("unit_test").should be_false
+      end
+    end
+
+    describe "get_definition" do
+      it "should return false" do
+        Rivet::Utils.get_definition("unit_test")
+      end
+    end
+
+    context "and with a group directory" do
+      before do
+        FileUtils.mkdir_p definition_dir
+      end
+
+      describe "load_definition" do
+        it "should return false" do
+          Rivet::Utils.load_definition("unit_test").should be_false
+        end
+      end
+
+      context "and with a conf.yml" do
+        before do
+          FileUtils.mkdir_p definition_dir
+          File.open(File.join(definition_dir,"conf.yml"),'w') do |f|
+            f.write(unit_test_definition_hash.to_yaml)
+          end
+        end
+        describe "load_definition" do
+          it "returns a hash" do
+            loaded_def = Rivet::Utils.load_definition("unit_test")
+            unit_test_definition_hash.each_pair { |k,v| loaded_def.should include(k => v) }
+          end
+        end
+        context "and with a defaults.yml" do
+          before do
+            File.open(File.join(Rivet::Utils::AUTOSCALE_DIR,"defaults.yml"),'w') do |f|
+              f.write(defaults_hash.to_yaml)
+            end
+          end
+
+          describe "consume_defaults" do
+            it "consume defaults returns a hash" do
+              results = Rivet::Utils.consume_defaults
+              defaults_hash.each_pair { |k,v| results.should include(k => v) }
+            end
+          end
+
+         describe "get_definition" do
+           it "returns a merged hash" do
+            result = Rivet::Utils.get_definition(definition_name)
+            merged_hash = defaults_hash.merge(unit_test_definition_hash)
+            result.should == defaults_hash.merge(unit_test_definition_hash)
+           end
+         end
+
+        end
+
+      end
+
+    end
+  end
+end
+