boucher 0.1

data/lib/boucher/provision.rb

@@ -0,0 +1,121 @@
+require 'boucher/compute'
+require 'boucher/io'
+require 'boucher/servers'
+require 'boucher/volumes'
+require 'boucher/nagios'
+require 'retryable'
+
+module Boucher
+  def self.each_required_server(&block)
+    Boucher::Config[:servers].each do |server_class|
+      server = find_server(server_class, Boucher::Config[:env])
+      block.yield(server, server_class)
+    end
+  end
+
+  def self.get_server(server_class, environment, state)
+    begin
+      Boucher::Servers.find(:env => environment.to_s, :class => server_class, :state => state)
+    rescue Boucher::Servers::NotFound
+      nil
+    end
+  end
+
+  def self.find_server(server_class, environment)
+    get_server(server_class, environment, "stopped") || get_server(server_class, environment, "running")
+  end
+
+  def self.establish_all_servers
+    Boucher.each_required_server do |server, server_class|
+      # Retries after 2, 4, 8, 16, 32, and 64 seconds
+      retryable(:tries => 6, :sleep => lambda { |n| 2**n }) do
+        # A RuntimeError will sometimes be thrown here, with a message of:
+        # "command failed with code 255"
+        Boucher.establish_server(server, server_class)
+      end
+    end
+  end
+
+  def self.establish_server(server, server_class)
+    if server.nil?
+      Boucher.provision(server_class, Boucher.server_classes[server_class.to_sym])
+    elsif server.state == "stopped"
+      Boucher::Servers.start(server.id)
+      server.reload
+      Boucher.cook_meals_on_server(server_class, Boucher.server_classes[server_class.to_sym], server)
+    else
+      Boucher.cook_meals_on_server(server_class, Boucher.server_classes[server_class.to_sym], server)
+    end
+  end
+
+  def self.provision(class_name, class_map)
+    puts "Provisioning new #{class_name} server..."
+    server = create_classified_server(class_map)
+    wait_for_server_to_boot(server)
+    wait_for_server_to_accept_ssh(server)
+    volumes = create_volumes(class_map, server)
+    attach_volumes(volumes, server)
+    cook_meals_on_server(class_name, class_map, server)
+    puts "\nThe new #{class_name} server has been provisioned! id: #{server.id}"
+  end
+
+  def self.attach_elastic_ips(class_name, server)
+    puts "Attaching elastic IPs..."
+    return unless Boucher::Config[:elastic_ips] && Boucher::Config[:elastic_ips][class_name]
+
+    puts "Associating #{server.id} with #{Boucher::Config[:elastic_ips][class_name]}"
+    compute.associate_address(server.id, Boucher::Config[:elastic_ips][class_name])
+  end
+
+  private
+
+  def self.cook_meals_on_server(class_name, class_map, server)
+    return unless class_map[:meals]
+
+    class_map[:meals].each do |meal|
+      meal = meal.call if meal.is_a?(Proc)
+      Boucher.cook_meal(server, meal)
+    end
+
+    attach_elastic_ips(class_name, server)
+  end
+
+  def self.wait_for_server_to_accept_ssh(server)
+    puts "Waiting for server's SSH port to open..."
+    Fog.wait_for { ssh_open?(server) }
+    puts
+  end
+
+  def self.wait_for_server_to_boot(server)
+    print "Waiting for server to boot..."
+    server.wait_for { print "."; server.ready? }
+    puts
+    Boucher.print_servers([server])
+  end
+
+  def self.create_classified_server(class_map)
+    server = compute.servers.new(:tags => {})
+    Boucher.classify(server, class_map)
+    server.save
+    Boucher.print_servers([server])
+    server
+  end
+
+  def self.create_volumes(class_map, server)
+    Array(class_map[:volumes]).map do |volume_name|
+      attributes = Boucher.volume_configs[volume_name]
+      snapshot = snapshots.get(attributes[:snapshot])
+      puts "Creating volume from snapshot #{snapshot.id}..."
+      Boucher::Volumes.create(server.availability_zone, snapshot, attributes[:device])
+    end
+  end
+
+  def self.attach_volumes(volumes, server)
+    volumes.each do |volume|
+      print "Attaching volume #{volume.id} to #{server.id}..."
+      Boucher::Volumes.attach(volume, server)
+      volume.wait_for { print "."; state == "in-use" }
+      puts
+    end
+  end
+end

data/lib/boucher/servers.rb

@@ -0,0 +1,92 @@
+require 'boucher/compute'
+
+module Boucher
+  module Servers
+    NotFound = Class.new(StandardError)
+
+    class << self
+      def instance
+        reload if !@instance
+        @instance
+      end
+
+      def reload
+        @instance = Boucher.compute.servers
+        @instance.each {} # Wake up you lazy list!
+        cultivate(@instance)
+      end
+
+      %w{all of_class in_env in_state search find [] with_id}.each do |m|
+        module_eval "def #{m}(*args); instance.#{m}(*args); end"
+      end
+
+      def cultivate(thing)
+        thing.extend(Boucher::Servers) if thing
+        thing
+      end
+    end
+
+    def all
+      self
+    end
+
+    def search(options={})
+      servers = self
+      servers = servers.of_class(options[:class]) if options[:class]
+      servers = servers.in_env(options[:env]) if options[:env]
+      servers = servers.in_state(options[:state]) if options[:state]
+      servers
+    end
+
+    def find(options={})
+      servers = search(options)
+      first = servers.first
+      if first.nil?
+        raise Boucher::Servers::NotFound.new("No server matches criteria: #{options.inspect}")
+      end
+      first
+    end
+
+    def in_env(env)
+      Servers.cultivate(self.find_all {|s| s.tags["Env"] == env.to_s })
+    end
+
+    def in_state(state)
+      Servers.cultivate(self.find_all {|s| s.state == state.to_s })
+    end
+
+    def of_class(klass)
+      Servers.cultivate(self.find_all {|s| s.tags["Class"] == klass.to_s })
+    end
+
+    def self.start(server_id)
+      Boucher.change_server_state(server_id, :start, "running")
+    end
+
+    def self.stop(server_id)
+      Boucher.change_server_state(server_id, :stop, "stopped")
+    end
+
+    def self.terminate(server)
+      Boucher::Nagios.remove_host(server)
+      volumes = server.volumes
+      volumes_to_destroy = volumes.select {|v| !v.delete_on_termination}
+
+      Boucher.change_server_state server.id, :destroy, "terminated"
+
+      volumes_to_destroy.each do |volume|
+        volume.wait_for { volume.state == 'available' }
+        puts "Destroying volume #{volume.id}..."
+        Boucher::Volumes.destroy(volume)
+      end
+    end
+
+    def with_id(server_id)
+      Servers.cultivate(self.find_all {|s| s.id == server_id}).first
+    end
+
+    def [](klass)
+      find(:env => Boucher::Config[:env], :class => klass, :state => "running")
+    end
+  end
+end

data/lib/boucher/storage.rb

@@ -0,0 +1,43 @@
+require 'boucher/env'
+require 'boucher/classes'
+require 'fog'
+
+module Boucher
+
+  S3_CONFIG = {
+          :provider => 'AWS',
+          :aws_secret_access_key => Boucher::Config[:aws_secret_access_key],
+          :aws_access_key_id => Boucher::Config[:aws_access_key_id]
+  }
+
+  def self.storage
+    @store ||= Fog::Storage.new(S3_CONFIG)
+    @store
+  end
+
+  module Storage
+
+    def self.list(dir_name)
+      dir = Boucher.storage.directories.get(dir_name)
+      result = dir.files.select {|f| f.key[-1] != "/" }.to_a
+      result
+    end
+
+    def self.put(dir_name, key, filename)
+      dir = Boucher.storage.directories.get(dir_name)
+      body = File.open(filename)
+      file = dir.files.new(:key => key, :body => body)
+      file.save
+      body.close
+      file
+    end
+
+    def self.get(dir_name, key, filename)
+      dir = Boucher.storage.directories.get(dir_name)
+      url = dir.files.get_https_url(key, Time.now + 3600)
+      Kernel.system("curl", url, "-o", filename)
+      dir.files.detect{|f| f.key == key}
+    end
+
+  end
+end

data/lib/boucher/tasks/console.rake

@@ -0,0 +1,21 @@
+require 'boucher/compute'
+require 'boucher/env'
+require 'boucher/io'
+require 'boucher/classes'
+require 'boucher/provision'
+require 'boucher/servers'
+require 'boucher/volumes'
+require 'boucher/nagios'
+
+desc "Starts a console with the Boucher modules loaded"
+task :console do
+  require 'pry'
+
+  Dir[File.expand_path("../../lib/boucher/**/*.rb", __FILE__)].each do |filename|
+    require filename
+  end
+
+  include Boucher
+
+  pry
+end

data/lib/boucher/tasks/env.rake

@@ -0,0 +1,42 @@
+require 'boucher/env'
+require 'boucher/provision'
+require 'retryable'
+
+namespace :env do
+
+  desc "Attaches elastic IPs"
+  task :elastic_ips, [:env] do |t, args|
+    Boucher.force_env!(args.env)
+    Boucher.each_required_server do |server, server_class|
+      Boucher.attach_elastic_ips(server_class, server)
+    end
+  end
+
+  desc "Starts and deploys all the servers for the specified environment"
+  task :start, [:env] do |t, args|
+    Boucher.force_env!(args.env)
+    Boucher.assert_env!
+    Boucher.establish_all_servers
+  end
+
+  desc "Stops all the servers for the specified environment"
+  task :stop, [:env] do |t, args|
+    Boucher.force_env!(args.env)
+
+    Boucher.each_required_server do |server, server_class|
+      if server
+        Boucher::Servers.stop(server.id)
+      else
+        puts "No #{server_class} server found for #{Boucher::Config[:env]} environment."
+      end
+    end
+  end
+
+  desc "Terminates all servers for the specified environment"
+  task :terminate, [:env] do |t, args|
+    Boucher.force_env!(args.env)
+    Boucher.each_required_server do |server, server_class|
+      Boucher::Servers.terminate(server) if server
+    end
+  end
+end

data/lib/boucher/tasks/nagios.rake

@@ -0,0 +1,74 @@
+require 'json'
+require 'fileutils'
+
+def make_config_path(host_name)
+  "/etc/nagios3/conf.d/#{host_name}.cfg"
+end
+
+def host_entry(host_name, ip)
+<<-HOST
+define host {
+  host_name             #{host_name}
+  alias                 #{host_name}
+  address               #{ip}
+  check_command         check_ssh
+  notification_interval 0
+  notification_period   24x7
+  max_check_attempts    10
+  notification_options  d,u,r
+  notifications_enabled 1
+}
+HOST
+end
+
+def service_entry(host_name, ip, plugin)
+<<-SERVICE
+define command {
+  command_name #{host_name}-#{plugin}
+  command_line /usr/lib/nagios/plugins/check_nrpe -H #{ip} -c "#{plugin}"
+}
+
+define service {
+  use                   generic-service
+  host_name             #{host_name}
+  check_command         #{host_name}-#{plugin}
+  service_description   #{plugin}
+  normal_check_interval 5
+  check_period          24x7
+  notifications_enabled 1
+}
+SERVICE
+end
+
+namespace :nagios do
+  desc "Remove a nagios host from this machine"
+  task :remove_host, [:name] do |t, args|
+    FileUtils.rm_f make_config_path(args.name)
+  end
+
+  desc "Adds a nagios host to be monitored by this machine"
+  task :add_host, [:name, :ip, :klass] do |t, args|
+    checks = Boucher::Nagios.checks_for(args.klass)
+    return if checks.empty?
+
+    File.open(make_config_path(args.name), "w") do |file|
+      host_name = "#{args.klass}-#{args.name}"
+
+      file.puts host_entry(host_name, args.ip)
+
+      checks.each do |plugin, command|
+        file.puts
+        file.puts service_entry(host_name, args.ip, plugin)
+      end
+    end
+  end
+
+  desc "Opens the nagios web console"
+  task :open do
+    server = Boucher::Servers["nagios_server"]
+    url = "http://#{server.public_ip_address}/nagios3"
+    puts "Nagios lives at #{url}"
+    puts "Login using nagiosadmin / 'we are many lonely souls'"
+    `open #{url}`
+  end
+end

data/lib/boucher/tasks/servers.rake

@@ -0,0 +1,127 @@
+require 'boucher/compute'
+require 'boucher/env'
+require 'boucher/io'
+require 'boucher/classes'
+require 'boucher/provision'
+require 'boucher/servers'
+require 'boucher/volumes'
+require 'boucher/nagios'
+require 'retryable'
+
+def meals
+  config_files = Dir.glob("config/*.json")
+  configs = config_files.map { |config_file| File.basename(config_file) }
+  configs.map { |config| config.gsub(".json", "") }
+end
+
+def server_listing(description, servers)
+  puts "Listing all AWS server #{description}..."
+  Boucher.print_servers servers
+  puts
+  puts "#{servers.size} server(s)"
+end
+
+namespace :servers do
+  desc "List all volumes for a given server"
+  task :volumes, [:server_id] do |t, args|
+    volumes = Boucher::Servers.with_id(args.server_id).volumes
+    Boucher.print_volumes(volumes)
+  end
+
+  desc "List ALL AWS servers, with optional [env] param."
+  task :list, [:env] do |t, args|
+    servers = Boucher::Servers.all
+    servers = args.env ? servers.in_env(args.env) : servers
+    server_listing("", servers)
+  end
+
+  desc "List AWS servers in specified environment"
+  task :in_env, [:env] do |t, args|
+    server_listing("in the '#{args.env}' environment", Boucher::Servers.in_env(args.env))
+  end
+
+  desc "List AWS servers in specified class"
+  task :of_class, [:klass] do |t, args|
+    server_listing("of class '#{args.klass}'", Boucher::Servers.of_class(args.klass))
+  end
+
+  desc "Terminates the specified server"
+  task :terminate, [:server_id] do |t, args|
+    server = Boucher::Servers.with_id(args.server_id)
+
+    if !server
+      puts "Server #{args.server_id} does not exist"
+      exit 1
+    end
+
+    begin
+      Boucher::Servers.terminate(server)
+    rescue => e
+      puts "\nTermination failed. This may be due to termination protection. If
+you're sure you wish to disable this protection, select the instance in the AWS
+web console and click Instance Actions -> Change Termination Protection -> Yes."
+      raise
+    end
+  end
+
+  desc "Stops the specified server"
+  task :stop, [:server_id] do |t, args|
+    server = Boucher.compute.servers.get(args.server_id)
+    Boucher::Nagios.remove_host(server)
+    Boucher::Servers.stop(args.server_id)
+  end
+
+  desc "Starts the specified server"
+  task :start, [:server_id] do |t, args|
+    Boucher::Servers.start(args.server_id)
+    server = Boucher.compute.servers.get(args.server_id)
+    Boucher::Nagios.add_host(server)
+  end
+
+  desc "Open an SSH session with the specified server"
+  task :ssh, [:server_id] do |t, args|
+    puts "Opening SSH session to #{args.server_id}"
+    server = Boucher.compute.servers.get(args.server_id)
+    Boucher.ssh server
+  end
+
+  desc "Download a file from the server"
+  task :download, [:server_id, :filepath] do |t, args|
+    puts "Downloading #{args.filepath}"
+
+    server      = Boucher.compute.servers.get(args.server_id)
+    remote_path = args.filepath
+    local_path  = File.expand_path(File.join("..", "..", File.basename(args.filepath)), __FILE__)
+
+    Boucher.download(server, remote_path, local_path)
+  end
+
+  desc "Fix permissions on key files"
+  task :key_fix do
+    system "chmod 0600 *.pem"
+  end
+
+  desc "Provision new server [#{Boucher.server_classes.keys.sort.join(', ')}]"
+  task :provision, [:klass] do |t, args|
+    class_map = Boucher.server_classes[args.klass.to_sym]
+    Boucher.provision(args.klass, class_map)
+  end
+
+  desc "Provision new, or chef existing server of the specified class"
+  task :establish, [:klass] do |t, args|
+    Boucher.assert_env!
+    server = Boucher.find_server(args.klass, ENV['BUTCHER_ENV'])
+    Boucher.establish_server(server, args.klass)
+  end
+
+  namespace :chef do
+    meals.each do |s_class|
+      desc "Cook #{s_class} meal"
+      task s_class, [:server_id] do |t, args|
+        Boucher.assert_env!
+        server = Boucher.compute.servers.get(args.server_id)
+        Boucher.cook_meal(server, s_class)
+      end
+    end
+  end
+end

data/lib/boucher/tasks/storage.rake

@@ -0,0 +1,32 @@
+require 'boucher/storage'
+require 'boucher/io'
+
+namespace :storage do
+
+  desc "Lists all the files in the infrastructure bucket on S3"
+  task :list do
+    files = Boucher::Storage.list("infrastructure")
+    Boucher.print_files files
+  end
+
+  desc "Puts a file in the infrastructure bucket"
+  task :put, [:file] do |t, args|
+    filename = args.file
+    key = File.basename(filename)
+    puts "Storing file #{filename} as #{key}"
+    file = Boucher::Storage.put("infrastructure", key, filename)
+    Boucher.print_files [file]
+    puts "File uploaded as #{key}."
+  end
+
+  desc "Gets a file from the infrastructure bucket. The file arg is the key on AWS."
+  task :get, [:file] do |t, args|
+    key = args.file
+    filename = File.basename(key)
+    puts "Getting file #{key} and saving to #{filename}"
+    file = Boucher::Storage.get("infrastructure", key, filename)
+    Boucher.print_files [file]
+    puts "File saved locally as #{filename}."
+  end
+
+end

data/lib/boucher/tasks/volumes.rake

@@ -0,0 +1,14 @@
+require 'boucher/volumes'
+require 'boucher/io'
+
+namespace :volumes do
+  desc "List provisioned volumes"
+  task :list do
+    Boucher.print_volumes(Boucher::Volumes.all)
+  end
+
+  desc "Destroy the specified volume"
+  task :destroy, [:volume_id] do |t, args|
+    Boucher.destroy_volume(args.volume_id)
+  end
+end

data/lib/boucher/tasks.rb

@@ -0,0 +1,6 @@
+
+task_dir = File.expand_path(File.join(File.dirname(__FILE__), "tasks"))
+
+Dir.glob(File.join(task_dir, "*.rake")).each do |lib|
+  load lib
+end

data/lib/boucher/util.rb

@@ -0,0 +1,9 @@
+module Boucher
+
+  def self.current_user
+    `git config user.name`.strip
+  rescue
+    "unknown"
+  end
+
+end

data/lib/boucher/volumes.rb

@@ -0,0 +1,41 @@
+require 'boucher/compute'
+
+module Boucher
+  def self.destroy_volume(volume_id)
+    volume = Boucher::Volumes.with_id(volume_id)
+    Volumes.destroy(volume)
+  end
+
+  module Volumes
+    def self.all
+      @volumes ||= Boucher.compute.volumes
+    end
+
+    def self.destroy(volumes)
+      Array(volumes).each do |volume|
+        volume.reload
+        volume.destroy
+      end
+    end
+
+    def self.with_id(volume_id)
+      all.find {|volume| volume.id == volume_id}
+    end
+
+    def self.create(zone, snapshot, device)
+      response  = Boucher.compute.create_volume(zone, snapshot.volume_size.to_i, snapshot.id)
+      volume_id = response.body["volumeId"]
+      volume    = Boucher.compute.volumes.get(volume_id)
+
+      volume.wait_for { ready? }
+      volume.device = device
+      volume
+    end
+
+    def self.attach(volumes, server)
+      Array(volumes).each do |volume|
+        Boucher.compute.attach_volume(server.id, volume.id, volume.device)
+      end
+    end
+  end
+end

data/spec/boucher/classes_spec.rb

@@ -0,0 +1,35 @@
+require_relative "../spec_helper"
+require 'boucher/classes'
+require 'ostruct'
+
+describe "Boucher Server Classes" do
+
+  before do
+    Boucher.stub(:current_user).and_return("joe")
+    @server = OpenStruct.new
+  end
+
+  it "pull classification from json" do
+    json = "{\"boucher\": {\"foo\": 1,\n \"bar\": 2}}"
+    Boucher.json_to_class(json).should == {:foo => 1, :bar => 2}
+  end
+
+  it "can classify base server" do
+    some_class = {:class_name => "base",
+                  :meals => ["base"]}
+    Boucher::Config[:default_instance_flavor_id] = 'm1.small'
+    Boucher::Config[:default_instance_groups] = ["SSH"]
+    Boucher.classify(@server, some_class)
+
+    @server.image_id.should == Boucher::Config[:base_image_id]
+    @server.flavor_id.should == 'm1.small'
+    @server.groups.should == ["SSH"]
+    @server.key_name.should == "test_key"
+    @server.tags["Class"].should == "base"
+    @server.tags["Name"].should_not == nil
+    @server.tags["Creator"].should == "joe"
+    @server.tags["Env"].should == Boucher::Config[:env]
+  end
+
+end
+