provisionator 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 cmaitchison
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# Provisionator
+
+Launch EC2 instances with a user data script that:
+  - mounts EBS volumes
+  - installs Ruby 2.0
+  - installs Chef and executes a chef-solo recipes
+  - deploys the latest release of code upon provisioning, capistrano style.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'provisionator'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install provisionator
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/provisionator.rb

@@ -0,0 +1,3 @@
+require 'provisionator/version'
+
+require 'provisionator/ec2_launcher'

data/lib/provisionator/capistrano/elasticsearch.rb

@@ -0,0 +1,7 @@
+Capistrano::Configuration.instance(:must_exist).load do
+  namespace :elasticsearch do    
+    task :restart, :roles => :elasticsearch do
+      run "sudo /usr/bin/monit restart elasticsearch"
+    end
+  end
+end

data/lib/provisionator/capistrano/provision.rb

@@ -0,0 +1,5 @@
+Capistrano::Configuration.instance(:must_exist).load do
+  task :provision do
+    run "/home/ubuntu/provision.sh"
+  end
+end

data/lib/provisionator/capistrano/sidekiq.rb

@@ -0,0 +1,28 @@
+Capistrano::Configuration.instance.load do
+  before "deploy:update_code", "sidekiq:quiet"
+  after "deploy:stop",    "sidekiq:stop"
+  after "deploy:start",   "sidekiq:start"
+  before "deploy:restart", "sidekiq:restart"
+
+  namespace :sidekiq do    
+    desc "Quiet sidekiq (stop accepting new work)"
+    task :quiet, :roles => :app, :on_no_matching_servers => :continue do
+      run "/usr/sbin/service sidekiq quiet"
+    end
+
+    desc "Stop sidekiq"
+    task :stop, :roles => :app, :on_no_matching_servers => :continue do
+      run "sudo /usr/bin/monit stop sidekiq"
+    end
+
+    desc "Start sidekiq"
+    task :start, :roles => :app, :on_no_matching_servers => :continue do
+      run "sudo /usr/bin/monit start sidekiq"
+    end
+
+    desc "Restart sidekiq"
+    task :restart, :roles => :app, :on_no_matching_servers => :continue do
+      run "sudo /usr/bin/monit restart sidekiq"
+    end
+  end
+end

data/lib/provisionator/capistrano/unicorn.rb

@@ -0,0 +1,19 @@
+Capistrano::Configuration.instance(:must_exist).load do
+  namespace :deploy do    
+    task :start, :roles => :app, :except => { :no_release => true } do
+      run "sudo /usr/bin/monit start unicorn"
+    end
+
+    task :stop, :roles => :app, :except => { :no_release => true } do
+      run "sudo /usr/bin/monit stop unicorn"
+    end
+
+    task :reload, :roles => :app, :except => { :no_release => true } do
+      run "sudo /usr/bin/monit reload unicorn"
+    end
+
+    task :restart, :roles => :app, :except => { :no_release => true } do
+      run "sudo /usr/bin/monit restart unicorn"
+    end
+  end
+end

data/lib/provisionator/ec2_launcher.rb

@@ -0,0 +1,106 @@
+require 'aws-sdk'
+require 'provisionator/ec2_userdata'
+
+module Provisionator
+  class Ec2Launcher
+  
+    def self.launch options={}
+      new(options).launch
+    end
+
+    def initialize options={}
+      options[:instance_type] ||= ENV['P_INSTANCE_TYPE'] || 'm1.small'
+      options[:no_release] ||= ENV['P_NO_RELEASE']
+      options[:ec2_region] ||= ENV['P_EC2_REGION'] || 'us-east-1'
+      options[:ec2_endpoint] ||= ENV['P_EC2_ENDPOINT'] || "ec2.#{options[:ec2_region]}.amazonaws.com"
+      options[:ami_id] ||= ENV['P_AMI_ID']
+      options[:ebs_volumes] ||= []
+      options[:key_name] ||= ENV['P_KEY_NAME']
+      options[:security_groups] ||= ENV['P_SECURITY_GROUPS'].try(:split, ',')
+      options[:availability_zones] ||= ENV['P_AVAILABILITY_ZONES'].try(:split, ',') || ["us-east-1a","us-east-1b","us-east-1c"]
+      options[:instance_count] ||= ENV['P_INSTANCE_COUNT'].try(:to_i) || 1
+      options[:load_balancer_name] ||= ENV['P_LOAD_BALANCER_NAME']
+    
+      [:app_name, :chef_node_definition, :ami_id, :key_name, :security_groups].each do |option|
+        raise "option ':#{option}' is required" if options[option].nil?
+      end
+      @options = options
+    end
+
+    def launch     
+      puts "Launching #{@options[:instance_count]} #{@options[:instance_type]} instance/s..." 
+
+      instances = launch_instances
+      wait_for_instances instances          
+      register_with_load_balancer instances
+
+      instances.each do |instance|
+        puts "Launched instance #{instance.id}, status: #{instance.status}, public dns: #{instance.dns_name}, public ip: #{instance.ip_address}"
+      end
+    end
+
+    private
+
+    def register_with_load_balancer instances
+      if @options[:load_balancer_name]
+        puts "Registering with load balancer..."
+        elb.register_instances_with_load_balancer(
+          load_balancer_name: @options[:load_balancer_name],
+          instances: instances.map{|i| {instance_id: i.id}}
+        )
+      end
+    end
+
+    def wait_for_instances instances
+      puts "Waiting for instances to boot..."
+      sleep 1 until instances.all?{|instance| instance.status != :pending}
+    end
+
+    def launch_instances
+      (1..@options[:instance_count]).map do
+        availability_zone = @options[:availability_zones].rotate!.first
+        instance_options = {
+          image_id: @options[:ami_id],
+          availability_zone: availability_zone,
+          key_name: @options[:key_name],
+          security_groups: @options[:security_groups],
+          instance_type: @options[:instance_type],
+          user_data: user_data,
+          count: 1
+        }
+        instance_options.merge!(block_device_mappings: block_device_mappings) if block_device_mappings.keys.any?
+        puts instance_options.inspect
+        instance = ec2.instances.create(instance_options)
+
+        instance.tags['Name'] = "#{@options[:chef_node_definition]}-#{availability_zone.split('').last}-#{instance.id}"
+        puts "Launching instance '#{instance.id}' in availability_zone '#{availability_zone}...'"
+        instance
+      end
+    end
+
+    def ec2
+      @ec2 ||= AWS::EC2.new(:ec2_endpoint => @options[:ec2_endpoint])
+    end
+
+    def elb
+      @elb ||= AWS::ELB::Client.new(:ec2_endpoint => @options[:ec2_endpoint])
+    end
+
+    def user_data attributes={}
+      @user_data ||= Provisionator::Ec2Userdata.new(@options).render
+    end
+
+    def block_device_mappings
+      @block_device_mappings ||= begin
+        available_device_mappings = ("/dev/sdf".."/dev/sdp").to_a
+        @options[:ebs_volumes].inject({}) do |mappings, ebs_volume|
+          mappings[available_device_mappings.shift] = { 
+            volume_size: ebs_volume[:size],
+            delete_on_termination: ebs_volume[:delete_on_termination]
+          }
+          mappings
+        end
+      end
+    end
+  end
+end

data/lib/provisionator/ec2_userdata.rb

@@ -0,0 +1,132 @@
+require 'erb'
+
+module Provisionator
+  class Ec2Userdata
+    def initialize options={}
+      [:app_name, :chef_node_definition, :chef_repo, :config_repo, :app_repo].each do |option|
+        raise "option ':#{option}' is required" if options[option].nil?
+      end
+
+      options[:chef_solo_rb_path] ||= "/home/ubuntu/.#{options[:app_name]}/provision/chef/solo.rb"
+      options[:chef_node_definition_base_file] ||= "/home/ubuntu/.#{options[:app_name]}/provision/chef/base.json"
+      options[:chef_node_definition_file] ||= "/home/ubuntu/.#{options[:app_name]}/provision/chef/#{options[:chef_node_definition]}.json"
+      options[:ruby_patch] ||= 'p195'
+      options[:ebs_mount_paths] = ebs_mount_paths(options[:ebs_volumes]) if options[:ebs_volumes]
+      @options = options
+    end
+
+
+    def ebs_mount_paths ebs_volumes
+      available_device_mappings = ("/dev/xvdf".."/dev/xvdp").to_a
+      ebs_volumes.map do |ebs_volume|
+        [available_device_mappings.shift, ebs_volume[:mount_location]]
+      end
+    end
+
+    def render
+      ERB.new(template).result(binding)
+    end
+
+    def template
+<<-TEMPLATE
+#!/bin/bash -xe
+exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+
+<% if @options[:ebs_mount_paths] %>
+  <% @options[:ebs_mount_paths].each do |device, mount_point| %>
+    mkfs.ext4 <%= device %>
+    mkdir -p -m 000 <%= mount_point %>
+    echo "<%= device %> <%= mount_point %> auto noatime 0 0" >> /etc/fstab
+    mount <%= device %> <%= mount_point %>
+  <% end %>
+<% end %>
+
+apt-get update
+apt-get upgrade -y
+ntpdate ntp.ubuntu.com
+export LC_CTYPE=en_US.UTF-8
+export LANG=en_US.UTF-8
+unset LC_ALL
+
+echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
+echo "UserKnownHostsFile=/dev/null" >> /etc/ssh/ssh_config
+echo "gem: --no-ri --no-rdoc" >> /etc/skel/.gemrc
+echo "gem: --no-ri --no-rdoc" >> /root/.gemrc
+echo "gem: --no-ri --no-rdoc" >> /home/ubuntu/.gemrc
+echo "gem: --no-ri --no-rdoc" >> /etc/gemrc
+
+#install ruby
+RUBY_PATCH=<%= @options[:ruby_patch] %>
+function install_ruby {
+  apt-get -y install build-essential zlib1g-dev libssl-dev libreadline-dev libyaml-dev libcurl4-openssl-dev
+  cd /tmp
+  wget ftp://ftp.ruby-lang.org/pub/ruby/2.0/ruby-2.0.0-$RUBY_PATCH.tar.gz
+  tar -xvzf ruby-2.0.0-$RUBY_PATCH.tar.gz
+  cd ruby-2.0.0-$RUBY_PATCH/
+  ./configure --prefix=/usr/local
+  make
+  make install
+}
+ruby -v | grep 2.0.0$RUBY_PATCH || install_ruby
+
+#install ruby 2.0 compatible version of chef
+cd /tmp
+curl -o chef.tar.gz -L https://api.github.com/repos/opscode/chef/tarball/CHEF-3935
+tar -xvzf chef.tar.gz
+cd opscode-chef-634ad58
+gem build chef.gemspec
+gem install chef-11.4.0.gem
+gem install ruby-shadow
+mkdir -p /var/cache/chef
+chown ubuntu:ubuntu /var/cache/chef
+
+apt-get install git-core -y
+
+gem install bundler
+
+cd /home/ubuntu
+
+<% if @options[:ssh_key] %>
+  mkdir -p /home/ubuntu/.ssh
+  echo "<%= @options[:ssh_key] %>" > /home/ubuntu/.ssh/id_rsa
+  chown -R ubuntu:ubuntu /home/ubuntu/.ssh
+  chmod 400 /home/ubuntu/.ssh/id_rsa
+<% end %>
+
+APP_NAME=<%= @options[:app_name] %>
+APP_PATH=<%= @options[:app_path] || "/u/apps/#{@options[:app_name]}" %>
+
+su ubuntu -c 'git clone <%= @options[:chef_repo] %> /home/ubuntu/.chef'
+su ubuntu -c 'git clone <%= @options[:config_repo] %> /home/ubuntu/.config'
+su ubuntu -c 'git clone <%= @options[:app_repo] %> /home/ubuntu/.<%= @options[:app_name] %>'
+
+ln -s /home/ubuntu/.config/s3search/production/app.sh /etc/profile.d/$APP_NAME.sh
+
+. /etc/profile.d/$APP_NAME.sh
+
+chef-solo -c <%= @options[:chef_solo_rb_path] %> -j <%= @options[:chef_node_definition_base_file] %> -l debug
+
+<% unless @options[:no_release] %>
+  mkdir -p $APP_PATH $APP_PATH/releases $APP_PATH/shared $APP_PATH/shared/system $APP_PATH/shared/log $APP_PATH/shared/pids $APP_PATH/shared/sockets
+  chmod g+w $APP_PATH $APP_PATH/releases $APP_PATH/shared $APP_PATH/shared/system $APP_PATH/shared/log $APP_PATH/shared/pids $APP_PATH/shared/sockets
+  chown -R ubuntu:ubuntu /u/
+  su ubuntu -c "git clone -q git@github.com:sodalis/s3search.git $APP_PATH/shared/cached-copy"
+  $(cd $APP_PATH/shared/cached-copy && APP_REVISION=$(git rev-parse HEAD))
+  cp -RPp $APP_PATH/shared/cached-copy $APP_PATH/releases/20130516070710
+  echo $APP_REVISION > $APP_PATH/releases/20130516070710/REVISION
+  ln -s $APP_PATH/releases/20130516070710 $APP_PATH/current
+  rm -R $APP_PATH/current/log 
+  ln -s $APP_PATH/shared/log/ $APP_PATH/current/log 
+  mkdir $APP_PATH/current/tmp
+  ln -s $APP_PATH/shared/pids/ $APP_PATH/current/tmp/pids
+  bundle install --gemfile $APP_PATH/releases/20130516070710/Gemfile --path $APP_PATH/shared/bundle --deployment --without development test
+  chown -R ubuntu:ubuntu /u/
+<% end %>
+
+echo <%= @options[:chef_node_definition] %> > /home/ubuntu/.chef_node_definition
+chown ubuntu:ubuntu /home/ubuntu/.chef_node_definition
+chef-solo -c <%= @options[:chef_solo_rb_path] %> -j <%= @options[:chef_node_definition_file] %> -l debug
+TEMPLATE
+    end
+  end
+end

data/lib/provisionator/version.rb

@@ -0,0 +1,3 @@
+module Provisionator
+  VERSION = "0.0.1"
+end

data/provisionator.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'provisionator/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "provisionator"
+  spec.version       = Provisionator::VERSION
+  spec.authors       = ["cmaitchison"]
+  spec.email         = ["cmaitchison@gmail.com"]
+  spec.description   = %q{To the cloud!}
+  spec.summary       = %q{To the cloud!}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'aws-sdk'
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: provisionator
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- cmaitchison
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: To the cloud!
+email:
+- cmaitchison@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/provisionator.rb
+- lib/provisionator/capistrano/elasticsearch.rb
+- lib/provisionator/capistrano/provision.rb
+- lib/provisionator/capistrano/sidekiq.rb
+- lib/provisionator/capistrano/unicorn.rb
+- lib/provisionator/ec2_launcher.rb
+- lib/provisionator/ec2_userdata.rb
+- lib/provisionator/version.rb
+- provisionator.gemspec
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: To the cloud!
+test_files: []