boucher 0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+.idea

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "template"]
+	path = template
+	url = git@github.com:8thlight/boucher_template.git

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm use 1.9.3-p125@boucher --create
+

data/Gemfile

@@ -0,0 +1,7 @@
+source :rubygems
+
+gem 'fog'
+gem 'pry'
+gem 'rake'
+gem 'retryable'
+gem 'rspec', "~>2.10"

data/Gemfile.lock

@@ -0,0 +1,53 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    builder (3.1.3)
+    coderay (1.0.7)
+    diff-lcs (1.1.3)
+    excon (0.16.3)
+    fog (1.6.0)
+      builder
+      excon (~> 0.14)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.0.4)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
+      ruby-hmac
+    formatador (0.2.3)
+    jruby-pageant (1.1.1)
+    method_source (0.8)
+    mime-types (1.19)
+    multi_json (1.3.6)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.6.0)
+      jruby-pageant (>= 1.1.1)
+    nokogiri (1.5.5)
+    pry (0.9.10)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.3.1)
+    rake (0.9.2.2)
+    retryable (1.3.1)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.3)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.3)
+    ruby-hmac (0.4.0)
+    slop (3.3.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fog
+  pry
+  rake
+  retryable
+  rspec (~> 2.10)

data/LICENSE

@@ -0,0 +1,9 @@
+Copyright (c) 2012 8th Light, Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,82 @@
+<img src="" alt="boucher logo" title="Boucher" align="right"/>
+# Boucher
+
+...
+
+## Getting Started
+
+
+### Creating a base image
+
+1. Launch new instance: Ubuntu Server 12.04.1 LTS
+    * Create a new key saved in your infrastructure project
+    * Be sure to add a security group that opens port 22 for SSH
+
+2. Update config/env/shared.rb
+    * :aws_key_filename - name of the .pem file you just created and saved in the project root
+    * :aws_region - which AWS region did you use?
+    * :aws_access_key_id and aws_secret_access_key - available in the AWS Management Console under Security Credentials
+
+3. List servers
+
+    rake servers:list
+
+4. SSH into new server.  (:username config must be 'ubunutu' at this point)
+
+    rake servers:ssh[<instance id>]
+
+5. Create new poweruser (unless you like 'unubutu' as your poweruser).
+
+    sudo adduser <username>
+    sudo adduser <username> sudo
+    sudo mkdir /home/<username>/.ssh
+    sudo cp .ssh/authorized_keys /home/<username>/.ssh/
+    sudo chown -R <username>:<username> /home/<username>/.ssh
+
+6. Logout.  Update config :username. Log back in.
+
+    rake servers:ssh[<instance id>]
+
+7. Delete the ubuntu user.
+
+    deluser ubuntu
+
+8. Enable sudo without typing password
+
+    sudo visudo
+    # add the following line at the end of the file:
+    <username> ALL=(ALL) NOPASSWD: ALL
+
+9. Install required pacakges and gems.
+
+    sudo apt-get update
+    sudo apt-get install ruby1.9.1 ruby1.9.1-dev git gcc make libxml2-dev libxslt1-dev
+    sudo apt-get upgrade
+    sudo gem install bundler chef
+
+10. Checkout your infrstructure repo.  (Yes.  You should push your repo even in this early stage.)
+If you use github, you'll have to generate ssh keys and add them to the github repo.
+
+    cd ~/.ssh
+    ssh-keygen -t rsa -C "your_email@youremail.com"
+    # Copy id_rsa.pub to your github user's ssh keys
+    cd ..
+    git clone git@github.com:<github account name>/<your infratructure project name>.git infrastructure
+
+10. Customize to your liking.
+
+  * install your preferred vim dot files
+  * etc...
+
+
+
+
+
+## Usage
+
+...
+
+## License
+
+Copyright (c) 2012 8th Light, Inc.
+MIT License

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'rspec/core/rake_task'
+
+desc "Run unit specs"
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = "./spec/**/*_spec.rb"
+end
+
+desc "Default: run the specs."
+task :default => :spec

data/boucher.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name        = "boucher"
+  s.version     = "0.1"
+  s.authors     = ["'Micah Micah'"]
+  s.email       = ["'micah@8thlight.com'"]
+  s.homepage    = "http://github.com/8thlight/boucher"
+  s.summary     = "AWS system deployment and management"
+  s.description = "AWS system deployment and management"
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = []
+  s.require_paths = ["lib"]
+  s.autorequire = 'boucher/tasks'
+
+  s.add_dependency('rake', '>= 0.9.2.2')
+  s.add_dependency('fog', '>= 1.6.0')
+  s.add_dependency('retryable', '>= 1.3.1')
+  s.add_dependency('pry', '>= 0.9.10')
+end

data/lib/boucher/ci.rb

@@ -0,0 +1,18 @@
+module Boucher
+  module CI
+    def self.each_server(&block)
+      threads = []
+      Boucher.each_required_server do |server, server_class|
+        thread = Thread.new { block.yield(server, server_class) }
+        threads << thread
+      end
+      threads.each { |t| t.join }
+    end
+
+    def self.terminate_server(server_class)
+      server = Boucher.get_server(server_class, "ci", "running")
+      Boucher.change_server_state server.id, :destroy, "terminated" if server
+    end
+  end
+end
+

data/lib/boucher/classes.rb

@@ -0,0 +1,39 @@
+require 'boucher/env'
+require 'boucher/util'
+require 'json'
+
+module Boucher
+
+  def self.json_to_class(json)
+    parser = JSON.parser.new(json, :symbolize_names => true)
+    config = parser.parse
+    config[:boucher]
+  end
+
+  def self.server_classes
+    if @server_classes.nil?
+      @server_classes = {}
+      Dir.glob(File.join("config", "*.json")).each do |file|
+        spec = json_to_class(IO.read(file))
+        class_name = File.basename(file).to_sym
+        @server_classes[class_name] = spec
+      end
+    end
+    @server_classes
+  end
+
+  def self.classify(server, class_spec)
+    server.image_id = class_spec[:base_image_id] || Boucher::Config[:base_image_id]
+    server.flavor_id = class_spec[:flavor_id] || Boucher::Config[:default_instance_flavor_id]
+    server.groups = class_spec[:groups] || Boucher::Config[:default_instance_groups]
+    server.key_name = class_spec[:key_name] || Boucher::Config[:aws_key_filename]
+    server.tags = {}
+    server.tags["Name"] = "#{class_spec[:class_name] || "base"} #{Time.new.strftime("%Y%m%d%H%M%S")}"
+    server.tags["Class"] = class_spec[:class_name] || "base"
+    server.tags["CreatedAt"] = Time.new.strftime("%Y%m%d%H%M%S")
+    server.tags["Creator"] = current_user
+    server.tags["Env"] = Boucher::Config[:env]
+    server.tags["Volumes"] = class_spec[:volumes]
+  end
+
+end

data/lib/boucher/compute.rb

@@ -0,0 +1,93 @@
+require 'boucher/env'
+require 'boucher/classes'
+require 'boucher/io'
+require 'fog'
+require 'retryable'
+
+module Boucher
+
+  EC2_CONFIG = {
+          :provider => 'AWS',
+          :aws_secret_access_key => Boucher::Config[:aws_secret_access_key],
+          :aws_access_key_id => Boucher::Config[:aws_access_key_id],
+          :region => Boucher::Config[:aws_region]
+  }
+
+  def self.compute
+    @compute ||= Fog::Compute.new(EC2_CONFIG)
+    @compute
+  end
+
+  def self.snapshots
+    @snapshots ||= compute.snapshots
+  end
+
+  def self.ssh(server, command=nil)
+    command_arg = nil
+    if command
+      command_arg = "\"#{command}\""
+    end
+
+    command = "#{ssh_command} #{Boucher::Config[:username]}@#{server.dns_name} #{command_arg}"
+    verbose command
+    Kernel.system command
+    raise "command failed with code #{$?.exitstatus}" unless $?.success?
+  end
+
+  def self.ssh_command
+    "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i #{Boucher::Config[:aws_key_filename]}.pem"
+  end
+
+  def self.download(server, remote_filepath, local_filepath)
+    command = ["rsync", "-azb", "-e", ssh_command, "--delete-after", "#{Boucher::Config[:username]}@#{server.dns_name}:#{remote_filepath}", local_filepath]
+    Kernel.system(*command)
+    raise "command failed with code #{$?.exitstatus}: #{command.inspect}" unless $?.success?
+  end
+
+  def self.rsync(server, from, to)
+    command = ["rsync", "-azb", "-e", ssh_command, "--delete-after",
+      from, "#{Boucher::Config[:username]}@#{server.dns_name}:#{to}"]
+    Kernel.system(*command)
+    raise "command failed with code #{$?.exitstatus}: #{command.inspect}" unless $?.success?
+  end
+
+  def self.update_recipes(server)
+    puts "Updating recipes on #{server.id}"
+    ssh server, "cd infrastructure && git checkout . && git clean -d -f && git pull && bundle"
+
+    %w[cookbooks config tasks].each do |folder|
+      rsync server, "#{folder}/", "infrastructure/#{folder}/"
+    end
+  end
+
+  def self.cook_meal(server, meal)
+    Boucher::Nagios.remove_host(server)
+
+    update_recipes(server)
+    ssh server, "cd infrastructure && sudo BUTCHER_ENV=#{Boucher::Config[:env]} BRANCH=#{Boucher::Config[:branch]} chef-solo -c config/solo.rb -j config/#{meal}.json"
+
+    Boucher::Nagios.add_host(server)
+  end
+
+  def self.ssh_open?(server)
+    ssh server, "echo 'SSH is open for business!'"
+    true
+  rescue Exception => e
+    false
+  end
+
+  def self.change_server_state(server_id, command, new_state)
+    print "#{command}-ing server #{server_id}..."
+    server = compute.servers.get(server_id)
+    server.send(command.to_sym)
+    server.wait_for { print "."; state == new_state }
+    puts
+    Boucher.print_servers [server]
+    puts
+    puts "The server has been #{command}-ed."
+  end
+
+  def self.find_servers
+    compute.servers
+  end
+end

data/lib/boucher/env.rb

@@ -0,0 +1,46 @@
+module Boucher
+
+  Config = {
+          :branch => ENV["BRANCH"] || "master"
+  } unless defined?(Boucher::Config)
+
+  def self.env_name
+    ENV["BUTCHER_ENV"] ? ENV["BUTCHER_ENV"] : :dev
+  end
+
+  env_dir = File.expand_path("config/env")
+  valid_envs = Dir[File.join(env_dir, "**", "*.rb")].map { |path| File.basename(path, ".rb") }
+  env_path = File.expand_path("#{env_name}.rb", env_dir)
+
+  unless defined?(Boucher::NO_LOAD_CONFIG)
+    unless File.exists?(env_path)
+      raise "Config file #{env_path} doesn't exist.\nYou need to change your BUTCHER_ENV environment variable to a valid environment name.\nValid environments: #{valid_envs.join(", ")}"
+    end
+
+    load env_path
+  end
+
+  def self.force_env!(name)
+    env = name.to_sym
+    Boucher::Config[:branch] = ENV["BRANCH"] || "master"
+    load File.expand_path(File.dirname(__FILE__) + "/../../config/env/shared.rb") unless defined?(Boucher::NO_LOAD_CONFIG)
+    file = File.expand_path(File.dirname(__FILE__) + "/../../config/env/#{env}.rb")
+    unless defined?(Boucher::NO_LOAD_CONFIG)
+      if File.exist?(file)
+        load file
+      else
+        require_relative "../../config/env/shared"
+      end
+    end
+  end
+
+  def self.assert_env!
+    unless ENV['BUTCHER_ENV']
+      raise 'BUTCHER_ENV must be set before running this command'
+    end
+
+    unless ENV['BRANCH']
+      raise 'BRANCH must be set before running this command'
+    end
+  end
+end

data/lib/boucher/io.rb

@@ -0,0 +1,108 @@
+module Boucher
+
+  module IO
+
+    def self.mock!
+      $stdout = StringIO.new
+      $stderr = StringIO.new
+      $stdin = StringIO.new
+    end
+
+    def self.real!
+      $stdout = STDOUT
+      $stderr = STDERR
+      $stdin = STDIN
+    end
+
+  end
+
+  def self.verbose(*args)
+    if ENV["VERBOSE"] != "false"
+      puts *args
+    end
+  end
+
+  SERVER_TABLE_FORMAT = "%-12s  %-12s  %-10s  %-10s  %-10s  %-15s  %-15s %-10s\n"
+
+  def self.print_server_table_header
+    puts
+    printf SERVER_TABLE_FORMAT, "ID", "Environment", "Class", "Creator", "State", "Public IP", "Private IP", "Inst. Size"
+    puts ("-" * 120)
+  end
+
+  def self.print_server(server)
+    printf SERVER_TABLE_FORMAT,
+           server.id,
+           (server.tags["Env"] || "???")[0...12],
+           (server.tags["Class"] || "???")[0...10],
+           (server.tags["Creator"] || "???")[0...10],
+           server.state,
+           server.public_ip_address,
+           server.private_ip_address,
+           server.flavor_id
+  end
+
+  def self.print_servers(servers)
+    print_server_table_header
+    sorted_servers = servers.sort_by{|s| [s.tags["Env"] || "?",
+                                          s.tags["Class"] || "?"]}
+    sorted_servers.each do |server|
+      print_server(server) if server
+    end
+    puts
+  end
+
+  def self.print_volumes(volumes)
+    id_sizes = []
+    size_sizes = []
+    state_sizes = []
+    zone_sizes = []
+    snapshot_sizes = []
+
+    Array(volumes).each do |volume|
+      id_sizes << volume.id.length
+      size_sizes << volume.size.to_s.length
+      state_sizes << volume.state.length
+      zone_sizes << volume.availability_zone.length
+      snapshot_sizes << volume.snapshot_id.to_s.length
+    end
+
+    id_length = id_sizes.max + 5
+    size_length = size_sizes.max + 5
+    state_length = state_sizes.max + 5
+    zone_length = zone_sizes.max + 5
+    snapshot_length = snapshot_sizes.max
+
+    puts "ID#{" "*(id_length - 2)}Size#{" "*(size_length - 4)}State#{" "*(state_length - 5)}Zone#{" "*(zone_length - 4)}Snapshot"
+    puts "-"*(id_length + size_length + state_length + zone_length + snapshot_length)
+
+    Array(volumes).each do |volume|
+      puts "#{volume.id}#{" "*(id_length - volume.id.length)}#{volume.size}GB#{" "*(size_length - volume.size.to_s.length - 2)}#{volume.state}#{" "*(state_length - volume.state.length)}#{volume.availability_zone}#{" "*(zone_length - volume.availability_zone.length)}#{volume.snapshot_id}#{" "*(snapshot_length - volume.snapshot_id.to_s.length)}"
+    end
+  end
+
+  FILE_TABLE_FORMAT = "%-60s  %-10s  %-25s  %-32s\n"
+
+  def self.print_file_table_header
+    puts
+    printf FILE_TABLE_FORMAT, "Key", "Size", "Last Modified", "etag"
+    puts ("-" * 150)
+  end
+
+  def self.print_file(file)
+    printf FILE_TABLE_FORMAT,
+           file.key,
+           file.content_length,
+           file.last_modified,
+           file.etag
+  end
+
+  def self.print_files(files)
+    print_file_table_header
+    files.each do |file|
+      print_file(file) if file
+    end
+    puts
+  end
+
+end

data/lib/boucher/nagios.rb

@@ -0,0 +1,50 @@
+require 'boucher/servers'
+require 'boucher/compute'
+
+module Boucher
+  module Nagios
+    def self.remove_host(host)
+      return if host.tags["Class"] == "nagios_server"
+
+      monitors_for(host.tags["Env"]).each do |monitor|
+        commands = [
+          "cd /home/#{Boucher::Config[:username]}/infrastructure",
+          "sudo rake nagios:remove_host[#{host.id}]",
+          "sudo /etc/init.d/nagios3 restart"
+        ]
+        Boucher.ssh(monitor, commands.join(" && "))
+      end
+    end
+
+    def self.add_host(host)
+      return if checks_for(host.tags["Class"]).empty?
+
+      monitors_for(host.tags["Env"]).each do |monitor|
+        commands = [
+          "cd /home/#{Boucher::Config[:username]}/infrastructure",
+          "sudo rake nagios:add_host[#{host.id},#{host.public_ip_address},#{host.tags["Class"]}]",
+          "sudo /etc/init.d/nagios3 restart"
+        ]
+
+        Boucher.ssh(monitor, commands.join(" && "))
+      end
+    end
+
+    def self.checks_for(server_class)
+      config_path = File.expand_path("../../../config/#{server_class}.json", __FILE__)
+      return [] unless File.exists? config_path
+      server_configuration = JSON.parse(File.read config_path)
+      server_configuration["nagios"] || []
+    end
+
+    private
+
+    def self.monitors_for(env)
+      Boucher::Servers.all.select do |monitor|
+        monitor.tags["Class"] == "nagios_server" &&
+        monitor.tags["Env"] == env &&
+        monitor.dns_name
+      end
+    end
+  end
+end