stemcell 0.0.11 → 0.2.0

data/README.md

@@ -1,28 +1,92 @@
-# Stemcell
+# Stemcell #
 
-Stemcell launches instances
+Stemcell launches instances in EC2.
+These instances are created to your specification, with knobs like AMI, instance type, and region exposed.
+The instances are bootstrapped with chef-solo, using a specified git repo and branch as the source of roles and recipes.
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
-    gem 'stemcell'
+```bash
+gem 'stemcell'
+```
 
 And then execute:
 
-    $ bundle
+```bash
+$ bundle
+```
 
 Or install it yourself as:
 
-    $ gem install stemcell
+```bash
+$ gem install stemcell
+```
+
+## Configuration
+
+You should create an rc file for stemcell with your standard options
+(and place it in the root dir as .stemcellrc?). You can see an example
+in examples/stemcellrc. You can get most of the options from your
+.chef/knife.rb but you will need to get the new chef deploy key so
+that instances that you launch can download code.
 
 ## Usage
 
+### Include your base config:
+
+```bash
+$ source .stemcellrc
+```
+
+### Simple launch:
+
+```bash
+$ ./bin/stemcell --chef-role $your_chef_role --git-branch $your_chef_branch
+```
+
+This will cause instance(s) to be launched and their ip's and instance
+id to be printed to the screen.
+
+### More options:
+
+```bash
+$ ./bin/stemcell --help
+```
+
+### Watching install:
+
+```bash
+$ ssh unbutu@$IP 'tail -f /var/log/init*'
+```
+
+### Terminating:
+
+This still needs to be completed. For now, you can kill using the
+amazon cli tools or the web ui.
+
+## Automation ##
+
+This README presents `stemcell` as a tool for administrators to use to create instances.
+However, we designed `stemcell` to be easily useful for automated systems which control server infrastructure.
+These automated systems can call out to `stemcell` on the command-line or use the ruby classes directly.
+
+To see how we manage our cloud, check out the rest of SmartStack, especially Cortex, our auto-scaling system.
+
+## Similar Tools ##
+
+There are a few additional tools which bootstrap EC2 instances with chef-solo.
+If you're using chef-server, obvious answer is [knife-ec2](https://github.com/opscode/knife-ec2).
+Unless you're working on a big team where lots of people edit cookbooks simultaneously, we strongly recommend this approach!
+It's especially excellent when paired with [hosted chef](http://www.opscode.com/hosted-chef/), which makes getting off the ground with configuration management fast and easy.
 
-## Contributing
+If you want to use knife-ec2 with chef-solo, you could use [knife solo](http://matschaffer.github.com/knife-solo/).
+Another approach which is great for interactive usage involves [using fabric to bootstrap chef](http://unfoldthat.com/2012/06/02/quick-deploy-chef-solo-fabric.html)([with gist](https://gist.github.com/va1en0k/2859812)).
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+Finally, we couldn't resist doing a bit of code archeology.
+People have been using chef with EC2 for a long time!
+One early article is [this one](http://web.archive.org/web/20110404114025/http://probablyinteractive.com/2009/3/29/Amazon%20EC2%20+%20Chef%20=%20Mmmmm.html), which isn't even on the web anymore.
+However, it's spawned some recently-active tools like [this](https://github.com/conormullen/chef-bootstrap) and [this](https://github.com/grempe/chef-solo-bootstrap).
+Similar approaches are mentioned [here](http://www.opinionatedprogrammer.com/2011/06/chef-solo-tutorial-managing-a-single-server-with-chef/), with code [herh](https://github.com/ciastek/ubuntu-chef-solo) or [here](https://github.com/riywo/ubuntu-chef-solo) (with accompanying [blog post](http://weblog.riywo.com/post/35976125760))
+[This article](http://illuminatedcomputing.com/posts/2012/02/simple-chef-solo-tutorial/), also mentions many worthwhile predecessors.

data/bin/stemcell

@@ -30,22 +30,22 @@ END_OF_BANNER
       :default => ENV['REGION'] ? ENV['REGION'] : 'us-east-1',
       )
 
-  opt('machine_type',
-      'instance type to launch',
+  opt('instance_type',
+      'machine type to launch',
       :type => String,
-      :default => ENV['MACHINE_TYPE'] ? ENV['MACHINE_TYPE'] : 'm1.small',
+      :default => ENV['INSTANCE_TYPE'] ? ENV['INSTANCE_TYPE'] : 'm1.small',
       )
 
-  opt('image',
+  opt('image_id',
       'ami to use for launch',
       :type => String,
-      :default => ENV['AMI'] ? ENV['AMI'] : 'ami-d726abbe',
+      :default => ENV['IMAGE_ID'] ? ENV['IMAGE_ID'] : 'ami-d726abbe',
       )
 
-  opt('security_group',
-      'security group to launch instance with',
+  opt('security_groups',
+      'security groups to launch instance with',
       :type => String,
-      :default => ENV['SECURITY_GROUP'] ? ENV['SECURITY_GROUP'] : 'default',
+      :default => ENV['SECURITY_GROUPS'] ? ENV['SECURITY_GROUPS'] : 'default',
       )
 
   opt('availability-zone',
@@ -55,7 +55,7 @@ END_OF_BANNER
       )
 
   opt('tags',
-      'tags to apply to image',
+      'comma-separated list of key=value pairs to apply',
       :type => String,
       :default => ENV['TAGS'],
       )
@@ -78,6 +78,12 @@ END_OF_BANNER
       :default => ENV['CHEF_ROLE'],
       )
 
+  opt('chef_environment',
+      'chef environment in which this instance will run',
+      :type => String,
+      :default => ENV['CHEF_ENVIRONMENT'],
+      )
+
   opt('git_origin',
       'git origin to use',
       :type => String,
@@ -87,7 +93,7 @@ END_OF_BANNER
   opt('git_branch',
       'git branch to run off',
       :type => String,
-      :default => ENV['GIT_BRANCH'],
+      :default => ENV['GIT_BRANCH'] ? ENV['GIT_BRANCH'] : 'production',
       )
 
   opt('git_key',
@@ -98,8 +104,8 @@ END_OF_BANNER
 
   opt('count',
       'number of instances to launch',
-      :type => String,
-      :default => ENV['COUNT'],
+      :type => Integer,
+      :default => ENV['COUNT'] ? ENV['COUNT'] : 1,
       )
 
 end
@@ -108,6 +114,8 @@ required_parameters = [
                        'aws_access_key',
                        'aws_secret_key',
                        'chef_role',
+                       'chef_environment',
+                       'chef_data_bag_secret',
                        'git_branch',
                        'git_key',
                        'git_origin',
@@ -120,6 +128,7 @@ by the #{arg.upcase.gsub('-','_')} environment variable" if
     options[arg].nil? or ! options[arg]
 end
 
+
 # convert tags from string to ruby hash
 tags = {}
 if options['tags']
@@ -130,6 +139,30 @@ if options['tags']
 end
 options['tags'] = tags
 
-stemcell = Stemcell::Stemcell.new(options)
-stemcell.launch({'count' => options['count']})
 
+# convert security_groups from comma seperated string to ruby array
+options['security_groups'] = options['security_groups'].split(',')
+
+
+# create stemcell object
+stemcell = Stemcell::Stemcell.new({
+  'aws_access_key' => options['aws_access_key'],
+  'aws_secret_key' => options['aws_secret_key'],
+  'region'         => options['region'],
+})
+
+
+# launch instance(s)
+stemcell.launch({
+  'instance_type'         => options['instance_type'],
+  'image_id'             => options['image_id'],
+  'security_groups'      => options['security_groups'],
+  'chef_role'            => options['chef_role'],
+  'chef_environment'     => options['chef_environment'],
+  'chef_data_bag_secret' => options['chef_data_bag_secret'],
+  'git_branch'           => options['git_branch'],
+  'git_key'              => options['git_key'],
+  'git_origin'           => options['git_origin'],
+  'key_name'             => options['key_name'],
+  'count'                => options['count'],
+})

data/examples/stemcellrc

@@ -0,0 +1,10 @@
+# these options need to be set
+export AWS_ACCESS_KEY=your_aws_access_key
+export AWS_SECRET_KEY=your_aws_secret_key
+export KEY_NAME=your_aws_ssh_key_name
+export CHEF_DATA_BAG_SECRET=/path/to/encrypted/data/bag/secret
+export GIT_KEY=/path/to/chef/deploy/key
+
+# these options should not need to be set
+export SECURITY_GROUP=default
+export GIT_ORIGIN=git@github.com:airbnb/chef.git

data/lib/stemcell/templates/bootstrap.sh.erb

@@ -31,12 +31,14 @@ exec 2> /var/log/init.err
 
 
 chef_version=11.4.0
-repo_dir=/etc/chef/src
-role=<%= @chef_role %>
-origin=<%= @git_origin %>
-branch=<%= @git_branch %>
-git_key='<%= @git_key_contents %>'
-data_bag_secret='<%= @chef_data_bag_secret %>'
+chef_dir=/etc/chef
+converge=/usr/local/bin/converge
+role=<%= opts['chef_role'] %>
+environment=<%= opts['chef_environment'] %>
+origin=<%= opts['git_origin'] %>
+branch=<%= opts['git_branch'] %>
+git_key='<%= opts["git_key"] %>'
+data_bag_secret='<%= opts["chef_data_bag_secret"] %>'
 
 
 ##
@@ -69,8 +71,6 @@ install_chef() {
   if ! which chef-solo > /dev/null ; then
     echo installing chef via omnibus...
     curl -L --silent https://www.opscode.com/chef/install.sh | sudo bash -s -- -v $chef_version  1>&2
-    # TODO(mkr): this is a hack to get chef-solo-search working
-    /opt/chef/embedded/bin/gem install --no-ri --no-rdoc treetop
   else
     echo chef is already installed
   fi
@@ -78,80 +78,86 @@ install_chef() {
 
 
 update_repo() {
-  git_wrapper=/etc/chef/git_wrapper
-  local keyfile=/etc/chef/git_key
+  echo -e "$git_key" > $keyfile
+  chmod 0400 $keyfile
+
+  echo "ssh -i $keyfile -o StrictHostKeyChecking=no \$1 \$2" > $git_wrapper
+  chmod 0500 $git_wrapper
+
   mkdir -p $(dirname $repo_dir)
-  if [ -d $repo_dir ]; then
-    echo updating source...
-    (cd $repo_dir && GIT_SSH=$git_wrapper git fetch && git reset --hard origin/$branch && git clean -fdx)
-  else
-    echo -e "$git_key" > $keyfile
-    chmod 0400 $keyfile
-    echo "ssh -i $keyfile  -o StrictHostKeyChecking=no \$1 \$2" > $git_wrapper
-    chmod 0500 $git_wrapper
-    echo downloading cookbook repo...
+  if [ ! -d $repo_dir ]; then
+    echo "downloading cookbook repo..."
     GIT_SSH=$git_wrapper git clone --branch $branch --depth 1 $origin $repo_dir
+  else
+    echo "cookbook repo already exists"
   fi
-  echo done updating code
+
+  echo "done updating code"
 }
 
 
 configure_chef() {
-  echo configuring chef...
-  mkdir -p /etc/chef
-  if [ -f $repo_dir/roles/${branch}.rb ]; then
-    echo "setting runlist to role[$branch],role[$role]"
-    cat<<EOF>/etc/chef/solo.json
-{
-  "run_list": [
-    "role[$branch]",
-    "role[$role]"
-  ],
-  "branch": "$branch",
-  "role": "$role"
-}
-EOF
-  else
-    echo "setting runlist to role[$role]"
-    cat<<EOF>/etc/chef/solo.json
-{
-  "run_list": "role[$role]",
-  "branch": "$branch",
-  "role": "$role"
-}
-EOF
-  fi
+  echo "saving current branch ($branch), role ($role), and environment ($environment)"
+  echo "$branch" > $branchfile
+  chmod 644 $branchfile
+
+  echo "$role" > $rolefile
+  chmod 644 $rolefile
+
+  echo "$environment" > $envfile
+  chmod 644 $envfile
+
+  echo "configuring chef solo..."
+  cat<<EOF > ${chef_dir}/solo.rb
+repo_dir =    "${repo_dir}"
+cookbook_path "#{repo_dir}/cookbooks"
+role_path     "#{repo_dir}/roles"
+data_bag_path "#{repo_dir}/data_bags"
 
-  cat<<EOF>/etc/chef/solo.rb
 log_level        :info
 log_location     STDOUT
-cookbook_path "$repo_dir/cookbooks"
-role_path "$repo_dir/roles"
-data_bag_path "$repo_dir/data_bags"
-json_attribs '/etc/chef/solo.json'
 EOF
-  echo -e "$data_bag_secret" > /etc/chef/encrypted_data_bag_secret
-  echo chef configured
+
+  echo -e "$data_bag_secret" > ${chef_dir}/encrypted_data_bag_secret
+  echo "chef configured"
 }
 
 
 configure_converger() {
-cat<<EOF>/usr/local/bin/converge
-#!/bin/bash -e
+  cat<<EOF > $converge
+#!/bin/bash -eu
+
+repo="${repo_dir}"
+branch=\`cat ${branchfile}\`
+role=\`cat ${rolefile}\`
+env=\`cat ${envfile}\`
+
+cd \${repo}
+
+runlist="role[\${role}]"
+if [ -f "roles/\${env}.rb" ]
+then
+  runlist+=",role[\${env}]"
+fi
 
-echo doing a hard reset to origin/$branch
-cd $repo_dir
-GIT_SSH=$git_wrapper git fetch
-git reset --hard origin/$branch
+echo doing a hard reset to origin/\${branch}
+GIT_SSH=${git_wrapper} git fetch
+git reset --hard origin/\${branch}
 git clean -fdx
-chef-solo
+
+json="{\"role\": \"\${role}\", \"env\": \"\${env}\", \"branch\": \"\${branch}\"}"
+json_file=\`tempfile\`
+echo \$json > \$json_file
+trap "{ rm -f '\$json_file' ; }" EXIT
+
+echo "running chef-solo with runlist \$runlist and json \$json (in file \$json_file)"
+chef-solo -o "\${runlist}" -j \${json_file} "\$@"
 EOF
-  chmod 0544 /usr/local/bin/converge
+  chmod 544 $converge
 }
 
-
 configure_chef_daemon() {
-  cat<<EOF>/etc/init/chef-solo.conf
+  cat<<EOF > /etc/init/chef-solo.conf
 description     "chef-solo"
 author          "Martin Rhoads"
 start on networking
@@ -164,12 +170,11 @@ EOF
 
 
 run_chef() {
-  echo running chef-solo...
-  chef-solo 1>&2
-  echo done running chef-solo
+  echo "doing initial chef run..."
+  $converge 1>&2
+  echo "initial chef run complete"
 }
 
-
 start_chef_daemon() {
   start chef-solo
 }
@@ -179,8 +184,16 @@ start_chef_daemon() {
 ## main
 ##
 
+#some derived vars
+repo_dir=${chef_dir}/src
+keyfile=${chef_dir}/git_key
+envfile=${chef_dir}/environment
+rolefile=${chef_dir}/role
+branchfile=${chef_dir}/branch
+git_wrapper=${chef_dir}/git_wrapper
 
-echo starting chef bootstrapping...
+echo "starting chef bootstrapping..."
+mkdir -p ${chef_dir}
 update
 install curl
 install git
@@ -190,6 +203,7 @@ update_repo
 configure_chef
 configure_converger
 run_chef
+
 configure_chef_daemon
 # start_chef_daemon
 

data/lib/stemcell/version.rb

@@ -1,3 +1,3 @@
 module Stemcell
-  VERSION = "0.0.11"
+  VERSION = "0.2.0"
 end

data/lib/stemcell.rb

@@ -8,65 +8,81 @@ module Stemcell
   class Stemcell
     def initialize(opts={})
       @log = Logger.new(STDOUT)
+      @log.level = Logger::INFO unless ENV['DEBUG']
+      @log.debug "creating new stemcell object"
       @log.debug "opts are #{opts.inspect}"
       ['aws_access_key',
        'aws_secret_key',
        'region',
-       'machine_type',
-       'image',
-       'security_group',
-
-       'chef_role',
-       'git_branch',
-       'git_key',
-       'git_origin',
-       'key_name',
       ].each do |req|
         raise ArgumentError, "missing required param #{req}" unless opts[req]
         instance_variable_set("@#{req}",opts[req])
       end
 
-      @zone = opts.include?('availability_zone') ? opts['availability_zone'] : nil
       @ec2_url = "ec2.#{@region}.amazonaws.com"
       @timeout = 120
       @start_time = Time.new
 
-      @tags = {
-        'Name' => "#{@chef_role}-#{@git_branch}",
-        'Group' => "#{@chef_role}-#{@git_branch}",
-        'created_by' => ENV['USER'],
-        'stemcell' => VERSION,
-      }
-      @tags.merge!(opts['tags']) if opts['tags']
-
-      begin
-        @git_key_contents = File.read(@git_key)
-      rescue Object => e
-        @git_key_contents = @git_key  # assume content is passed in
-      end
-
-      if opts.include?('chef_data_bag_secret')
-        begin
-          @chef_data_bag_secret = File.read(opts['chef_data_bag_secret'])
-        rescue Object => e
-          @chef_data_bag_secret = opts['chef_data_bag_secret'] # assume secret is passed in
-        end
-      else
-        @chef_data_bag_secret = ''
-      end
-
       AWS.config({:access_key_id => @aws_access_key, :secret_access_key => @aws_secret_key})
       @ec2 = AWS::EC2.new(:ec2_endpoint => @ec2_url)
       @ec2_region = @ec2.regions[@region]
-      @user_data = render_template
     end
 
+
     def launch(opts={})
-      File.open('/tmp/user-data', 'w') {|f| f.write(@user_data) }
-      instances = do_launch(opts)
+      verify_required_options(opts,[
+        'image_id',
+        'security_groups',
+        'key_name',
+        'count',
+        'chef_role',
+        'chef_environment',
+        'chef_data_bag_secret',
+        'git_branch',
+        'git_key',
+        'git_origin',
+        'instance_type',
+      ])
+
+      # attempt to accept keys as file paths
+      opts['git_key'] = try_file(opts['git_key'])
+      opts['chef_data_bag_secret'] = try_file(opts['chef_data_bag_secret'])
+
+      # generate tags and merge in any that were specefied as in inputs
+      tags = {
+        'Name' => "#{opts['chef_role']}-#{opts['chef_environment']}",
+        'Group' => "#{opts['chef_role']}-#{opts['chef_environment']}",
+        'created_by' => ENV['USER'],
+        'stemcell' => VERSION,
+      }
+      tags.merge!(opts['tags']) if opts['tags']
+
+      # generate user data script to boot strap instance based on the
+      # opts that we were passed.
+      user_data = render_template(opts)
+
+      launch_options = {
+        :image_id => opts['image_id'],
+        :security_groups => opts['security_groups'],
+        :user_data => opts['user_data'],
+        :instance_type => opts['instance_type'],
+        :key_name => opts['key_name'],
+        :count => opts['count'],
+        :user_data => user_data,
+      }
+      launch_options.merge({:availability_zone => opts['availability_zone']}) if opts['availability_zone']
+
+      # launch instances
+      instances = do_launch(launch_options)
+
+      # wait for aws to report instance stats
       wait(instances)
-      set_tags(instances)
+
+      # set tags on all instances launched
+      set_tags(instances, tags)
+
       print_run_info(instances)
+      @log.info "launched instances successfully"
       return instances
     end
 
@@ -83,7 +99,7 @@ module Stemcell
     end
 
     def wait(instances)
-      @log.info "Waiting for #{instances.count} instances (#{instances.inspect}):"
+      @log.info "Waiting up to #{@timeout} seconds for #{instances.count} instances (#{instances.inspect}):"
 
       while true
         sleep 5
@@ -100,18 +116,18 @@ module Stemcell
       @log.info "all instances in running state"
     end
 
-    def do_launch(opts={})
-      options = {
-        :image_id => @image,
-        :security_groups => @security_group,
-        :user_data => @user_data,
-        :instance_type => @machine_type,
-        :key_name => @key_name,
-      }
-      options[:availability_zone] = @zone if @zone
-      options[:count] = opts['count'] if opts.include?('count')
+    def verify_required_options(params,required_options)
+      @log.debug "params is #{params}"
+      @log.debug "required_options are #{required_options}"
+      required_options.each do |required|
+        raise ArgumentError, "you need to provide option #{required}" unless params[required]
+      end
+    end
 
-      instances = @ec2_region.instances.create(options)
+    def do_launch(opts={})
+      @log.debug "about to launch instance(s) with options #{opts}"
+      @log.info "launching instances"
+      instances = @ec2_region.instances.create(opts)
       instances = [instances] unless instances.class == Array
       instances.each do |instance|
         @log.info "launched instance #{instance.instance_id}"
@@ -119,19 +135,21 @@ module Stemcell
       return instances
     end
 
-    def set_tags(instances=[])
+    def set_tags(instances=[],tags)
+      @log.info "setting tags on instance(s)"
       instances.each do |instance|
-        instance.tags.set(@tags)
+        instance.tags.set(tags)
       end
     end
 
-    def render_template
+    def render_template(opts={})
       this_file = File.expand_path __FILE__
       base_dir = File.dirname this_file
       template_file_path = File.join(base_dir,'stemcell','templates','bootstrap.sh.erb')
       template_file = File.read(template_file_path)
       erb_template = ERB.new(template_file)
       generated_template = erb_template.result(binding)
+      @log.debug "genereated template is #{generated_template}"
       return generated_template
     end
 
@@ -142,5 +160,15 @@ module Stemcell
         instance.delete
       end
     end
+
+    # attempt to accept keys as file paths
+    def try_file(opt="")
+      begin
+        return File.read(opt)
+      rescue Object => e
+        return opt
+      end
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stemcell
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-23 00:00:00.000000000 Z
+date: 2013-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: trollop
@@ -57,6 +57,7 @@ files:
 - README.md
 - Rakefile
 - bin/stemcell
+- examples/stemcellrc
 - lib/stemcell.rb
 - lib/stemcell/templates/bootstrap.sh.erb
 - lib/stemcell/version.rb