elzar 0.1.2 → 0.2.0

data/lib/elzar/compute.rb

@@ -0,0 +1,52 @@
+require 'elzar/fog'
+require 'slushy'
+require 'yaml'
+
+module Elzar
+  module Compute
+    def self.provision_and_bootstrap!(instance_name, aws_config)
+      instance_id, instance_ip = provision(instance_name, aws_config)
+      bootstrap(instance_id, aws_config)
+
+      [instance_id, instance_ip]
+    end
+
+    def self.provision(name, aws_config)
+      config = aws_config['server']['creation_config']
+      config['tags'] = {'Name' => name}
+
+      slushy_instance = Slushy::Instance.launch(fog_connection(aws_config), config)
+      [slushy_instance.instance_id, slushy_instance.server.public_ip_address]
+    end
+
+    def self.bootstrap(instance_id, aws_config)
+      slushy_instance = slushy_instance_for(instance_id, aws_config)
+      slushy_instance.bootstrap
+    end
+
+    def self.converge!(instance_id, aws_config)
+      tmpdir = Elzar.merge_and_create_temp_directory File.expand_path('provision/')
+      slushy_instance = slushy_instance_for(instance_id, aws_config)
+      slushy_instance.converge tmpdir
+
+      [slushy_instance.instance_id, slushy_instance.server.public_ip_address]
+    end
+
+    def self.destroy!(instance_id, aws_config)
+      slushy_instance = slushy_instance_for(instance_id, aws_config)
+      slushy_instance.terminate
+    end
+
+    private
+
+    def self.fog_connection(aws_config)
+      @fog_connection ||= Fog::Compute.new(aws_config['aws_credentials'].merge(:provider => 'AWS'))
+    end
+
+    def self.slushy_instance_for(instance_id, aws_config)
+      Slushy::Instance.new(fog_connection(aws_config), instance_id).tap do |s|
+        s.server.private_key = aws_config['server']['private_key']
+      end
+    end
+  end
+end

data/lib/elzar/core_ext/hash.rb

@@ -0,0 +1,18 @@
+module HashDeepMergeExt
+
+  # Implementation pulled from ActiveSupport
+  def deep_merge(other_hash)
+    dup.deep_merge!(other_hash)
+  end
+
+  def deep_merge!(other_hash)
+    other_hash.each_pair do |k,v|
+      tv = self[k]
+      self[k] = tv.is_a?(Hash) && v.is_a?(Hash) ? tv.deep_merge(v) : v
+    end
+    self
+  end
+
+end
+
+Hash.send(:include, HashDeepMergeExt) unless Hash.new.respond_to?(:deep_merge)

data/lib/elzar/fog.rb

@@ -0,0 +1,53 @@
+require 'fog'
+require 'fog/core/ssh'
+
+# Monkey-patch Fog 1.3.1 to stream SSH output
+# (in real time) to stdout.
+class Fog::SSH::Real
+  def run(commands)
+    commands = [*commands]
+    results  = []
+    begin
+      Net::SSH.start(@address, @username, @options) do |ssh|
+        commands.each do |command|
+          result = Fog::SSH::Result.new(command)
+          ssh.open_channel do |ssh_channel|
+            ssh_channel.request_pty
+            ssh_channel.exec(command) do |channel, success|
+              unless success
+                raise "Could not execute command: #{command.inspect}"
+              end
+
+              channel.on_data do |ch, data|
+                result.stdout << data
+                puts data
+              end
+
+              channel.on_extended_data do |ch, type, data|
+                next unless type == 1
+                result.stderr << data
+                puts data
+              end
+
+              channel.on_request('exit-status') do |ch, data|
+                result.status = data.read_long
+              end
+
+              channel.on_request('exit-signal') do |ch, data|
+                result.status = 255
+              end
+            end
+          end
+          ssh.loop
+          results << result
+        end
+      end
+    rescue Net::SSH::HostKeyMismatch => exception
+      exception.remember_host!
+      sleep 0.2
+      retry
+    end
+    results
+  end
+end
+

data/lib/elzar/ssh_key_locator.rb

@@ -0,0 +1,37 @@
+module Elzar
+  module SshKeyLocator
+
+    class << self
+      def default_key_file_paths
+        %w[id_dsa.pub id_ecdsa.pub id_rsa.pub].map do |base_filename|
+          File.expand_path("~/.ssh/#{base_filename}")
+        end
+      end
+
+      def find_keys(candidate_key_file_paths = default_key_file_paths)
+        local_keys = find_local_keys(candidate_key_file_paths)
+        local_keys.empty? ? find_agent_keys : local_keys
+      end
+
+      def find_local_keys(candidate_key_file_paths = default_key_file_paths)
+        first_existing_file = candidate_key_file_paths.find { |p| File.exist?(p) }
+        return [] unless first_existing_file
+
+        file_content = File.read(first_existing_file)
+        split_keys(file_content)
+      end
+
+      def find_agent_keys
+        keys = split_keys(`ssh-add -L`)
+        $?.success? ? keys : []
+      end
+
+      private
+
+      def split_keys(s)
+        s.split("\n").reject { |k| k.strip.empty? }
+      end
+    end
+
+  end
+end

data/lib/elzar/templates/Gemfile

@@ -5,8 +5,8 @@ group "development" do
   gem "json",                   "1.5.2"
 
   gem "elzar"
-  gem "vagrant",                "0.8.7"
-  gem "chef",                   "0.10.8"
-  gem "knife-solo",             "0.0.4"
-  gem "knife-github-cookbooks", :git => "git://github.com/sumbach/knife-github-cookbooks.git"
+  gem "vagrant",                "1.0.5"
+  gem "chef",                   "10.14.4"
+  gem "knife-solo",             "0.0.14"
+  gem "knife-github-cookbooks", "~> 0.1.7"
 end

data/lib/elzar/templates/Vagrantfile.erb

@@ -5,7 +5,7 @@ Vagrant::Config.run do |config|
   config.vm.box       = "lucid64"
   config.vm.box_url   = "http://files.vagrantup.com/lucid64.box"
   config.vm.host_name = <%= @vm_host_name.inspect %>
-  config.vm.network     "172.25.5.5"
+  config.vm.network     :hostonly, "172.25.5.5"
   config.vm.provision :shell, :path => "upgrade-chef.sh"
   config.vm.provision :chef_solo do |chef|
     chef.cookbooks_path = <%= @cookbooks_path.inspect %>

data/lib/elzar/templates/aws_config.private.yml

@@ -0,0 +1,13 @@
+# !!! DO NOT add this file to source control !!!
+#
+# Settings in this file override settings in aws_config.yml
+aws_credentials:
+  aws_access_key_id: <your aws access key id>
+  aws_secret_access_key: <your aws secret access key>
+
+server:
+  private_key: |
+    -----BEGIN RSA PRIVATE KEY-----
+    Include the RSA private key here. This should correspond to the keypair indicated
+    by server[:creation_config][:key_name] in aws_config.yml.
+    -----END RSA PRIVATE KEY-----

data/lib/elzar/templates/aws_config.yml

@@ -0,0 +1,6 @@
+server:
+  creation_config:
+    flavor_id: <instance type, e.g. 'm1.large'>
+    image_id: <AMI to bootstrap with; must be some Ubuntu server image; e.g., "ami-fd589594" for Ubuntu 11.04>
+    groups: <security group to place the new deployment in, e.g. "default">
+    key_name: <name of the public/private keypair to start instance with>

data/lib/elzar/templates/data_bags/deploy/authorized_keys.json

@@ -1,9 +1,8 @@
 {
   "id":"authorized_keys",
   "keys": [
-"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAzqe0G9iGh9QoCdBVYmaTIxZU0cZP15uiXtLUsOf+BAjniNy2tX6fM+9+Mwmz76sqPlpL/ai/eVKg/WlmjtZErInTUdEu5IGoNqORLtwxGtu4u0tl/bByIEeYInHqlFlQnyEFeSEcWwW9JKWC7jL3Wh4M8FZrrGIQ7yd+XF4Q3vUpL+MZThs3/+VSoGvqcPbRGHoLtlgjwsjobXI6ckj839SkT0zVPuIoUl7zxlD4FRH95Hv60fjVIFU3D2aeOJrLkqHFogGjrXG47AIjgw5Xb1cwTUA0zQQ1Lq5TF038smXEyvySPOAuPPTnBC11TuzADdEuI0zFmjSu2RYi4/TfZw== alex.redington@thinkrelevance.com",
-"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/aJwkhY3lpj6f62mpsy+fZ1lS90jF2cSTkX/bmjXrBO4yyFQsPFzsZhe6kdKkJGsVocppS8TZujzwL2vnhH4zfIIDIEwl9F9bqBznTwuwP0BcPIEJTEc7YVds2ocjR73A9YHHZxV7IfzYnYVunEmPtvPCSdQn+xq3V1bzY9tJJzXu/9DI0qxjTQAWB5nPfqyGwNnn2pcfNeZQquXMRHQnEyncpGq936vrYemwr25MlGNGzvhq0vQM8Qo5PSqh7XwSHR31aTsyL6RDF9nMOqHvqRsF+MKrsZSCYhjZ9ebuSRjdiDeWGNeYA8+zJJiirjJYwN6qxIGQ1AAkNrqZfKzV russolsen@thinkrelevance.com",
-"ssh-dss AAAAB3NzaC1kc3MAAACBAJCaIm17djv5DOGA0amRQTibubORv0mVCDnhkT/rYOIEAXr9S+s3JqrWqC9Q9++wFxVvuTr7rJArktoPQJXuAZfk9yH1Yyd2dhzMM/jHnw/TYKOnYj6mn4ygwARm9oZkuBI9AV81ICBtZPdBwPAH4vpOoY8FtKCCaiKrJdOCJfSPAAAAFQDlOSP4ehJY3w65d5GDEMftZ2va/wAAAIAUwmNmhG4W8LNv+ONsvJ/+GY2OumSJt72uzjcFRx9OLFzMgTG5YqQNuCbosA5SXvRS4w1F2+gPW/Tsdh9D3CElqOq3slMY+1JuDWznDK3byDe5sHQz8OOpjBtxFdn+bZJHTXercVpfIqjxR+903/zjYUT2tgv8kyZu44thn8+wRAAAAIBCoJqTsZwwErIH+RnX+LIETPYkKMt664CYb9mnNDj5a6hV/JX+ThiFdDd4YBUTVQkyNTQasJgOfZyauDT0BQJiEIiag1guvRtLhYH3PEId3vMsMGfv5349v2UqVr9g01CnbOL+yGc/Akxcw58/5QZe45S3A7hDbgUGsEBTNEuGaQ== sam@thinkrelevance.com",
-"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/kJu5qUgbtVqYdxxN0LvZMJ5//JRno7bgnQ8ZB6dAouyE13GaiJ3jl61xCdKIOe6FL9/x2C379fcj2MUXiq/iIH4bllgn9owux7TucE4bKmMm2DxwDH/A2kvES3V+7LznvSSyhUJTSIVII2gQdeSaeE1DrJDNJJv/riXvmP6BI1AaLJnfE82Ym7XuYeLqvz1f6pCwxCZipuxjqr8HNBL3MfVN4Dw2xxya5AwAiR5Za93fVmszXBoyGo+BaUTTjdGzb4ghGOHjgp7rdLWbRoVcyXWntXq0jQ/Jkdg0dUVvFgLA88/IjyKvstmI1SrKK6SlwCWLSjAESpX3xRG2FNhp gabriel.horner@gmail.com"
-   ]
+    "TODO: Add your public SSH keys for the provisioned server here",
+    "TODO: Add your public SSH keys for the provisioned server here",
+    "TODO: Add your public SSH keys for the provisioned server here"
+  ]
 }

data/lib/elzar/templates/dna/rails.json

@@ -0,0 +1,15 @@
+{
+  "run_list":["role[plumbing]", "role[postgres_database]", "ruby", "passenger", "rails_app", "ruby::path"],
+  "passenger":  {
+    "version":  "3.0.11",
+    "root_path":  "/opt/relevance-ruby/lib/ruby/gems/1.9.1/gems/passenger-3.0.11",
+    "module_path":  "/opt/relevance-ruby/lib/ruby/gems/1.9.1/gems/passenger-3.0.11/ext/apache2/mod_passenger.so"
+  },
+  "ruby":  {
+    "version":  "1.9.3-p194",
+    "url":  "http://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.3-p194.tar.gz"
+  },
+  "rails_app":  {
+    "name":  "TODO: Replace with the name of your Rails app"
+  }
+}

data/lib/elzar/templates/gitignore

@@ -0,0 +1 @@
+aws_config.private.yml

data/lib/elzar/version.rb

@@ -1,3 +1,3 @@
 module Elzar
-  VERSION = '0.1.2'
+  VERSION = '0.2.0'
 end

data/script/ci_nightly

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+source "$HOME/.rvm/scripts/rvm"
+export CI_RUBY_VERSION="ruby-1.9.3-p194"
+export CI_GEMSET=elzar_nightly
+
+# Tests and app are all running in CI_GEMSET
+export NO_RVM=true
+
+rvm use "$CI_RUBY_VERSION@$CI_GEMSET"
+gem install bundler
+bundle install
+
+bundle exec rspec --tag ci

data/spec/fixtures/rails_integration_template/add_root_user.rb

@@ -0,0 +1,9 @@
+class AddRootUser < ActiveRecord::Migration
+  def up
+    User.find_or_create_by_username! 'root'
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

data/spec/fixtures/rails_integration_template/database.yml

@@ -0,0 +1,7 @@
+production:
+  adapter: postgresql
+  encoding: unicode
+  database: elzar_nightly_app_production
+  pool: 5
+  username: deploy
+  password: d3pl0y-p0stgr3s

data/spec/fixtures/rails_integration_template/deploy.rb

@@ -0,0 +1,11 @@
+require 'bundler/capistrano'
+require 'capistrano/relevance/all'
+
+set :application, "elzar_nightly_app"
+set :repository, "/tmp/elzar_nightly_app" # TODO Find a way not to duplicate this path here and inside the spec. Pass as env arg?
+
+set(:server_ip) { ENV['SERVER_IP'] || raise("You must supply SERVER_IP") }
+
+role :web, server_ip
+role :app, server_ip
+role :db,  server_ip, :primary => true

data/spec/fixtures/rails_integration_template/template.rb

@@ -0,0 +1,22 @@
+TemplateRoot = File.expand_path '..', __FILE__
+
+gem 'capistrano', :group => 'deployment', :git => 'git://github.com/capistrano/capistrano', :ref => 'b31e2f5'
+gem 'capistrano-relevance', :group => 'deployment'
+
+gem 'therubyracer', :group => 'assets', :platforms => :ruby
+
+run 'bundle install'
+run 'bundle exec capify .'
+
+remove_file File.join('config', 'deploy.rb')
+copy_file File.join(TemplateRoot, 'deploy.rb'), File.join('config', 'deploy.rb')
+
+run 'rails generate scaffold user username:string'
+
+next_migration_timestamp = (Time.now + 1)
+migration_filename = next_migration_timestamp.utc.strftime("%Y%m%d%H%M%S") + '_add_root_user.rb'
+copy_file File.join(TemplateRoot, 'add_root_user.rb'), File.join('db', 'migrate', migration_filename)
+
+git :init
+git :add => "."
+git :commit => "-a -m 'Initial commit'"

data/spec/integration/rails_spec.rb

@@ -0,0 +1,190 @@
+require 'spec_helper'
+require 'fileutils'
+require 'json'
+
+describe "Rails integration", :ci => true do
+  let(:rails_app)         { 'elzar_nightly_app' }
+  let(:path_to_rails_app) { File.join '/tmp', rails_app }
+  let(:server_name)       { "Elzar Nightly (rails) - #{Time.now.strftime("%Y-%m-%d %H:%M:%S")}" }
+  let(:aws_config_dir)    { ENV['AWS_CONFIG_DIR'] || raise('You must set AWS_CONFIG_DIR to run the integration tests') }
+  let(:instance_info)     { { :id => nil, :ip => nil } }
+
+  ######################################################################
+  # Command line helpers
+  ######################################################################
+
+  # wrapper around system
+  def shell(cmd)
+    puts "Executing #{cmd}..."
+    Bundler.clean_system(cmd)
+    abort "Command '#{cmd}' failed" unless $?.success?
+  end
+
+  def elzar(args)
+    puts "Running `elzar #{args}`"
+
+    super
+
+    raise "Error running `elzar #{args}`. Failed with: #{stderr}" unless process.exitstatus == 0
+  end
+
+  def rake(cmd)
+    sh "bundle exec rake #{cmd}"
+  end
+
+  def in_rails_app(&block)
+    pwd = FileUtils.pwd
+    FileUtils.cd path_to_rails_app
+    yield
+  ensure
+    FileUtils.cd pwd
+  end
+
+  ######################################################################
+  # Rails app helpers
+  ######################################################################
+
+  def create_new_rails_app
+    shell "gem install rails"
+    FileUtils.rm_rf(path_to_rails_app)
+    rails_template = File.expand_path('../../fixtures/rails_integration_template/template.rb', __FILE__)
+    shell %Q{rails new "#{path_to_rails_app}" -d postgresql -m "#{rails_template}"}
+  end
+
+  def configure_elzar
+    in_rails_app do
+      rewrite_json('provision/dna.json') do |dna|
+        dna['rails_app']['name'] = rails_app
+      end
+    end
+  end
+
+  def rewrite_json(path_to_json, &block)
+    json = JSON.parse File.read(path_to_json)
+    yield json
+    File.open(path_to_json, 'w') { |f| f << JSON.generate(json) }
+  end
+
+  ######################################################################
+  # AWS helpers
+  ######################################################################
+
+  def aws_config
+    @aws_config ||= Elzar::AwsConfig.load_configs(aws_config_dir)
+  end
+
+  def fog
+    @fog ||= Fog::Compute.new(aws_config['aws_credentials'].merge(:provider => 'AWS'))
+  end
+
+  def server(instance_id)
+    fog.servers.get(instance_id).tap do |s|
+      s.private_key = aws_config['server']['private_key']
+    end
+  end
+
+  def ssh(server, cmd)
+    job = nil
+    capture_stdout { job = server.ssh(cmd).first }
+    job
+  end
+
+  def put_database_config_on_server(server)
+    shared_path = "/var/www/apps/#{rails_app}/shared/config"
+    path_to_db_config = File.expand_path('../../fixtures/rails_integration_template/database.yml', __FILE__)
+    server.scp path_to_db_config, "/home/ubuntu/database.yml"
+
+    ssh server, "sudo mkdir -p #{shared_path}"
+    ssh server, "sudo mv ~/database.yml #{shared_path}/database.yml"
+    ssh server, "sudo chown -R deploy:deploy #{shared_path}"
+  end
+
+  def destroy_instance(instance_id)
+    in_rails_app do
+      elzar "destroy \"#{instance_id}\" --aws_config_dir=#{aws_config_dir}"
+    end
+  end
+
+  def capture_instance_details(output)
+    id = output.match(/Instance ID: (.+)$/i)[1]
+    ip = output.match(/Instance IP: (.+)$/i)[1]
+    [id, ip]
+  end
+
+  ######################################################################
+  # Assertion helpers
+  ######################################################################
+
+  # Returns true if the command gives zero exit status, false for non zero exit status.
+  def execute_local_command(cmd)
+    Bundler.clean_system(cmd)
+  end
+
+  # Returns true if the command gives zero exit status, false for non zero exit status.
+  def execute_remote_command(server, cmd)
+    ssh(server, cmd).status == 0
+  end
+
+  def assert_state_after_init
+    in_rails_app do
+      execute_local_command('ls provision > /dev/null').should == true
+      execute_local_command('grep -q rails provision/dna.json').should == true
+    end
+  end
+
+  def assert_state_after_preheat(server)
+    execute_remote_command(server, 'gem list | grep chef').should == true
+  end
+
+  def assert_state_after_cook(server)
+    execute_remote_command(server, '/opt/relevance-ruby/bin/ruby -v | grep 1\.9\.3').should == true
+    execute_remote_command(server, 'sudo service postgresql status').should == true
+    execute_remote_command(server, 'sudo service nginx status').should == true
+    execute_remote_command(server, 'ls /home/deploy').should == true
+  end
+
+  def assert_state_after_deploy(server_ip)
+    execute_local_command(%Q{curl -sL -w '%{http_code}' #{server_ip} -o /dev/null | grep 200}).should == true
+    execute_local_command(%Q{curl -s #{server_ip}/users.json | grep '"username":"root"'}).should == true
+  end
+
+  it 'works' do
+    create_new_rails_app
+
+    in_rails_app do
+      elzar "init --dna=rails"
+    end
+
+    assert_state_after_init
+
+    in_rails_app do
+      configure_elzar
+      elzar %Q{preheat "#{server_name}" --aws_config_dir=#{aws_config_dir}}
+      instance_info[:id], instance_info[:ip] = capture_instance_details(stdout)
+    end
+
+    instance_info[:id].should_not == nil
+    instance_info[:ip].should_not == nil
+    server = server(instance_info[:id])
+
+    assert_state_after_preheat(server)
+
+    in_rails_app do
+      elzar "cook \"#{instance_info[:id]}\" --aws_config_dir=#{aws_config_dir}"
+    end
+
+    assert_state_after_cook(server)
+
+    in_rails_app do
+      shell %Q{SERVER_IP="#{instance_info[:ip]}" cap deploy:setup}
+      put_database_config_on_server(server)
+      shell %Q{SERVER_IP="#{instance_info[:ip]}" cap deploy}
+    end
+
+    assert_state_after_deploy(instance_info[:ip])
+  end
+
+  after(:each) do
+    destroy_instance(instance_info[:id]) if instance_info[:id]
+  end
+end