stemcell 0.2.9 → 0.4.4

data/.gitignore

@@ -19,3 +19,4 @@ tmp
 \#*\#
 .\#*
 .*sw?
+.DS_Store

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Martin Rhoads
+Copyright (c) 2013 Airbnb, Inc.
 
 MIT License
 
@@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -24,35 +24,49 @@ Or install it yourself as:
 $ gem install stemcell
 ```
 
+Or build the gem locally and install it:
+
+```bash
+$ gem build stemcell.gemspec
+$ gem install ./stemcell-0.4.3.gem
+```
+
 ## Configuration
 
-You should create an rc file for stemcell with your standard options
-(and place it in the root dir as .stemcellrc?). You can see an example
-in examples/stemcellrc. You can get most of the options from your
-.chef/knife.rb but you will need to get the new chef deploy key so
-that instances that you launch can download code.
+If you're using the command line tool, you'll need to add a `stemcell.json` to the root of your chef repo.
+It contains default attributes for launching instances as well as the mapping from backing stores to images.
+For an example, see `examples/stemcell.json`.
+
+You should create an RC file for stemcell with your standard options (and place it in the root dir as .stemcellrc?).
+You can see an example in `examples/stemcellrc`.
+As documented in that file, you will need:
+* a mono-repo for chef (like the kind described [here](https://github.com/opscode/chef-repo))
+* a local checkout of the repo
+* an ssh key that's allowed to read that repo
+* AWS credentials with permission to create instances
+* an AWS [ssh key pair](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html)
+* an encrypted data bag secret (optional)
 
 ## Usage
 
 ### Include your base config:
 
 ```bash
-$ source .stemcellrc
+$ source ~/.stemcellrc
 ```
 
 ### Simple launch:
 
 ```bash
-$ ./bin/stemcell --chef-role $your_chef_role --git-branch $your_chef_branch
+$ stemcell $your_chef_role --git-branch $your_chef_branch
 ```
 
-This will cause instance(s) to be launched and their ip's and instance
-id to be printed to the screen.
+This will cause instance(s) to be launched and their ip's and instance id to be printed to the screen.
 
 ### More options:
 
 ```bash
-$ ./bin/stemcell --help
+$ stemcell --help
 ```
 
 ### Watching install:
@@ -75,8 +89,6 @@ This README presents `stemcell` as a tool for administrators to use to create in
 However, we designed `stemcell` to be easily useful for automated systems which control server infrastructure.
 These automated systems can call out to `stemcell` on the command-line or use the ruby classes directly.
 
-To see how we manage our cloud, check out the rest of SmartStack, especially Cortex, our auto-scaling system.
-
 ## Similar Tools ##
 
 There are a few additional tools which bootstrap EC2 instances with chef-solo.

data/bin/necrosis

@@ -20,12 +20,20 @@ options = Trollop::options do
       :type => String,
       :default => ENV['AWS_SECRET_KEY']
       )
+
+  opt('aws_regions',
+      "comma-separated list of aws regions to search",
+      :type => String,
+      :default => ENV['AWS_REGIONS'],
+      :short => :r
+      )
+
+  opt :non_interactive, 'Do not ask confirmation', :short => :f
 end
 
-required_parameters = [
-                       'aws_access_key',
-                       'aws_secret_key',
-                      ]
+NON_INTERACTIVE = options[:non_interactive]
+
+required_parameters = %w(aws_access_key aws_secret_key)
 
 required_parameters.each do |arg|
   raise ArgumentError, "--#{arg.gsub('_','-')} needs to be specified on the commandline or set \
@@ -33,35 +41,38 @@ by the #{arg.upcase.gsub('-','_')} environment variable" if
     options[arg].nil? or ! options[arg]
 end
 
-raise ArgumentError, "you did not provide any instance ids to kill" if ARGV.size == 0
+raise ArgumentError, "you did not provide any instance ids to kill" if ARGV.empty?
 
 # a hash from instance_id => [ec2 instance objects]
 instances = ARGV.inject({}) { |h, n| h[n] = []; h }
 
-ec2 = AWS::EC2.new(:access_key_id => options['aws_access_key'], :secret_access_key => options['aws_secret_key'])
-ec2.regions.each do |region|
-  instances.each do |id, objects|
-    instance = region.instances[id]
-    objects << [instance, region] if instance.exists?
-  end
-end
+# convert comma-separated list to Array
+regions = (options['aws_regions'] || '').split(',')
 
-# kill the instances
-instances.each do |id, objects|
-  if objects.count == 0
-    puts "No instance #{id} found"
-    next
-  end
+def authorized? instance
+  return true if NON_INTERACTIVE
+
+  puts 'Terminate? (y/N)'
+  confirm = $stdin.gets
+  confirmed = confirm && confirm.chomp.downcase == 'y'
 
-  if objects.count > 1
-    puts "Found multiple instances named #{id}"
-    next
+  if confirmed
+    age = Time.now - instance.launch_time
+    days = age / (24*3600)
+    if days > 2
+      puts "Instance is #{days} days old. REALLY terminate? (y/N)"
+      confirm = $stdin.gets
+      confirmed = confirm && confirm.chomp.downcase == 'y'
+    end
   end
 
-  instance, region = objects[0]
+  return confirmed
+end
+
+def delete_instance id, instance, region
   if instance.api_termination_disabled?
     puts "Cannot terminate instance #{id} -- termination protection enabled"
-    next
+    return
   end
 
   puts "Instance #{id} (#{instance.status} in #{region.name})"
@@ -71,10 +82,33 @@ instances.each do |id, objects|
     puts "\t#{k} : #{v}"
   end
 
-  puts "Terminate? (y/N)? "
-  confirm = $stdin.gets
-  if confirm && confirm.chomp.downcase == 'y'
+  if authorized? instance
     instance.terminate
     puts "Instance #{id} terminated"
   end
 end
+
+AWS.memoize do
+  ec2 = AWS::EC2.new(:access_key_id => options['aws_access_key'], :secret_access_key => options['aws_secret_key'])
+  ec2.regions.each do |region|
+    next unless regions.empty? || regions.include?(region.name)
+    instances.each do |id, objects|
+      instance = region.instances[id]
+      objects << [instance, region] if instance.exists?
+    end
+  end
+
+  instances.each do |id, objects|
+    case objects.count
+    when 0
+      STDERR.puts "No instance #{id} found"
+      next
+    when 1
+      instance, region = objects.first
+      delete_instance id, instance, region
+    else
+      puts "Found multiple instances named #{id}"
+      next
+    end
+  end
+end

data/bin/stemcell

@@ -1,181 +1,24 @@
 #!/usr/bin/env ruby
-
-# -*- mode: shell -*- 
-
-require_relative '../lib/stemcell'
-
-require 'trollop'
-
-options = Trollop::options do
-  version "stemcell #{Stemcell::VERSION} (c) 2013 Airbnb"
-  banner  "Stemcell: get ready to rock"
-
-  opt('aws_access_key',
-      "aws access key",
-      :type => String,
-      :default => ENV['AWS_ACCESS_KEY']
-      )
-
-  opt('aws_secret_key',
-      "aws secret key",
-      :type => String,
-      :default => ENV['AWS_SECRET_KEY']
-      )
-
-  opt('region',
-      'ec2 region to launch in',
-      :type => String,
-      :default => ENV['REGION'] ? ENV['REGION'] : 'us-east-1',
-      )
-
-  opt('instance_type',
-      'machine type to launch',
-      :type => String,
-      :default => ENV['INSTANCE_TYPE'] ? ENV['INSTANCE_TYPE'] : 'm1.small',
-      )
-
-  opt('image_id',
-      'ami to use for launch',
-      :type => String,
-      :default => ENV['IMAGE_ID'] ? ENV['IMAGE_ID'] : 'ami-d726abbe',
-      )
-
-  opt('security_groups',
-      'security groups to launch instance with',
-      :type => String,
-      :default => ENV['SECURITY_GROUPS'] ? ENV['SECURITY_GROUPS'] : 'default',
-      )
-
-  opt('availability_zone',
-      'zone in which to launch instances',
-      :type => String,
-      :default => ENV['AVAILABILITY_ZONE'],
-      )
-
-  opt('tags',
-      'comma-separated list of key=value pairs to apply',
-      :type => String,
-      :default => ENV['TAGS'],
-      )
-
-  opt('key_name',
-      'aws ssh key name for the ubuntu user',
-      :type => String,
-      :default => ENV['KEY_NAME'],
-      )
-
-  opt('iam_role',
-      'IAM role to associate with the instance',
-      :type => String,
-      :default => ENV['IAM_ROLE'],
-      )
-
-  opt('ebs_optimized',
-      'launch an EBS-Optimized instance',
-      :type => :flag,
-      )
-
-  opt('chef_data_bag_secret',
-      'path to secret file (or the string containing the secret)',
-      :type => String,
-      :default => ENV['CHEF_DATA_BAG_SECRET'],
-      )
-
-  opt('chef_role',
-      'chef role of instance to be launched',
-      :type => String,
-      :default => ENV['CHEF_ROLE'],
-      )
-
-  opt('chef_environment',
-      'chef environment in which this instance will run',
-      :type => String,
-      :default => ENV['CHEF_ENVIRONMENT'],
-      )
-
-  opt('git_origin',
-      'git origin to use',
-      :type => String,
-      :default => ENV['GIT_ORIGIN'],
-      )
-
-  opt('git_branch',
-      'git branch to run off',
-      :type => String,
-      :default => ENV['GIT_BRANCH'] ? ENV['GIT_BRANCH'] : 'production',
-      )
-
-  opt('git_key',
-      'path to the git repo deploy key (or the string containing the key)',
-      :type => String,
-      :default => ENV['GIT_KEY'],
-      )
-
-  opt('count',
-      'number of instances to launch',
-      :type => Integer,
-      :default => ENV['COUNT'] ? ENV['COUNT'] : 1,
-      )
-
-end
-
-required_parameters = [
-                       'aws_access_key',
-                       'aws_secret_key',
-                       'chef_role',
-                       'chef_environment',
-                       'chef_data_bag_secret',
-                       'git_branch',
-                       'git_key',
-                       'git_origin',
-                       'key_name',
-                      ]
-
-required_parameters.each do |arg|
-  raise ArgumentError, "--#{arg.gsub('_','-')} needs to be specified on the commandline or set \
-by the #{arg.upcase.gsub('-','_')} environment variable" if
-    options[arg].nil? or ! options[arg]
-end
-
-
-# convert tags from string to ruby hash
-tags = {}
-if options['tags']
-  options['tags'].split(',').each do |tag_set|
-    key, value = tag_set.split('=')
-    tags[key] = value
-  end
-end
-options['tags'] = tags
-
-
-# convert security_groups from comma seperated string to ruby array
-options['security_groups'] = options['security_groups'].split(',')
-
-
-# create stemcell object
-stemcell = Stemcell::Stemcell.new({
-  'aws_access_key' => options['aws_access_key'],
-  'aws_secret_key' => options['aws_secret_key'],
-  'region'         => options['region'],
-})
-
-
-# launch instance(s)
-stemcell.launch({
-  'availability_zone'    => options['availability_zone'],
-  'instance_type'        => options['instance_type'],
-  'image_id'             => options['image_id'],
-  'security_groups'      => options['security_groups'],
-  'chef_role'            => options['chef_role'],
-  'chef_environment'     => options['chef_environment'],
-  'chef_data_bag_secret' => options['chef_data_bag_secret'],
-  'git_branch'           => options['git_branch'],
-  'git_key'              => options['git_key'],
-  'git_origin'           => options['git_origin'],
-  'key_name'             => options['key_name'],
-  'tags'                 => options['tags'],
-  'count'                => options['count'],
-  'iam_role'             => options['iam_role'],
-  'ebs_optimized'        => options['ebs_optimized'],
-})
+require 'stemcell'
+
+# Stemcell (c) 2012-2013 Airbnb
+#
+# This script starts an EC2 instance using attributes stored in the expanded
+# runlist of a given role, respecting default and overriden attributes. To use
+# stemcell with your role, specify a new attribute called "instance_metadata",
+# the options therein will get merged against the default options (stored in
+# the stemcell.json file in your local chef root), as well as any options
+# passed in the command line.
+#
+# Usage: stemcell [chef role] [options]
+#
+# The priority order for options (highest to lowest):
+#
+#   1) Arguments passed to launch on the command line
+#   2) Environment variables
+#   3) Options derived from the runlist
+#   4) Default instance store options (stemcell.json)
+#   5) Default launch options (stemcell.json)
+#   6) Base options (see below)
+
+Stemcell::CommandLine.run!

data/examples/stemcell.json

@@ -0,0 +1,26 @@
+{
+  "defaults": {
+    "region":"us-east-1",
+    "security_groups": ["default"],
+    "instance_type": "m1.small",
+    "backing_store": "instance_store"
+  },
+
+  "backing_store": {
+    "ebs": {
+      "image_id": "ami-23d9a94a"
+    },
+    "instance_store": {
+      "image_id": "ami-d9d6a6b0"
+    }
+  },
+
+  "availability_zones": {
+    "us-east-1": [
+      "us-east-1a",
+      "us-east-1b",
+      "us-east-1c",
+      "us-east-1e"
+    ]
+  }
+}

data/examples/stemcellrc

@@ -1,10 +1,29 @@
 # these options need to be set
 export AWS_ACCESS_KEY=your_aws_access_key
 export AWS_SECRET_KEY=your_aws_secret_key
+
+# AWC keys can be managed from the AWS console by going
+#   to EC2 and then clicking 'Key Pairs'. Every EC2 instance
+#   needs a key pair to launch; you can use the private component
+#   of this key pair to get into your machine before chef has run
 export KEY_NAME=your_aws_ssh_key_name
+
+# We use encrypted data bags; to generate an ecrypted data bag
+#   secret, follow instructions here:
+#   http://docs.opscode.com/chef/essentials_data_bags.html#secret-keys
 export CHEF_DATA_BAG_SECRET=/path/to/encrypted/data/bag/secret
+
+# This expects a standard opscode mono-repo, like the one
+#   modeled here: https://github.com/opscode/chef-repo
+export LOCAL_CHEF_ROOT=/path/to/your/local/chef/repository
+
+# The git origin is the publically-accessible version of the
+#   local chef root; we use a github enterprise server
+export GIT_ORIGIN=git@git.airbnb.com:airbnb/chef.git
+
+# The git key is an SSH key which has pull permissions on
+#   the GIT_ORIGIN repo
 export GIT_KEY=/path/to/chef/deploy/key
 
-# these options should not need to be set
+# Which security group should you launch
 export SECURITY_GROUP=default
-export GIT_ORIGIN=git@github.com:airbnb/chef.git

data/lib/stemcell/command_line.rb

@@ -0,0 +1,222 @@
+require 'aws/creds'
+require 'colored'
+
+module Stemcell
+  class CommandLine
+    attr_reader :launcher
+    attr_reader :showing_help
+    attr_reader :chef_root
+    attr_reader :chef_role
+
+    VERSION_STRING = "Stemcell (c) 2012-2013 Airbnb."
+    BANNER_STRING  = "Launch instances from metadata stored in roles!\n" \
+                     "Usage: stemcell [chef role] [options]"
+
+    OPTION_PARSER_DEFAULTS = {
+      'non_interactive' => false,
+      'aws_creds'       => 'default',
+      'ssh_user'        => 'ubuntu'
+    }
+
+    def self.run!
+      CommandLine.new.run!
+    end
+
+    def run!
+      determine_root_and_showing_help
+
+      initialize_launcher
+      retrieve_defaults
+      configure_option_parser
+
+      options = parse_options_or_print_help
+      determine_role(options)
+
+      print_usage_and_exit unless chef_role
+      # If we didn't successfully initialize the launcher, we should exit.
+      exit unless @launcher
+
+      print_version
+      display_chef_repo_warnings
+
+      # Retrieve AWS credentials via the aws-creds gems (if available)
+      awc_name = options.delete('aws_creds')
+      credentials = retrieve_credentials_from_awc(awc_name)
+      options.merge!(credentials) if credentials
+      # Move launcher in the non-interactive mode (if requested)
+      launcher.interactive = !options.delete('non_interactive')
+
+      instances = launch_instance(chef_role, options)
+      tail_converge(instances, options) if options['tail']
+
+      puts "\nDone.\n".green
+    end
+
+    private
+
+    def determine_root_and_showing_help
+      # First pass at parsing the options to find the local chef root and
+      # whether or not we're showing the help screen.
+      provisional_option_parser = OptionParser.new(:override_help => true)
+      initial_options = provisional_option_parser.parse!(ARGV.dup)
+
+      # Remove the role before parsing any remaining options.
+      @showing_help = !!initial_options['help']
+      @chef_root    =   initial_options['local_chef_root']
+
+      # If we didn't receive a chef root, assume the current directory.
+      @chef_root  ||=   File.expand_path('.')
+    end
+
+    def initialize_launcher
+      @launcher = MetadataLauncher.new(:chef_root => chef_root)
+    rescue NoTemplateError
+      puts "Couldn't find `stemcell.json` in the local chef repo.".red
+      puts "You must specify the root of the local checkout of your chef " \
+           "respository by using the --local-chef-root options or " \
+           "setting the LOCAL_CHEF_ROOT environment variable."
+    rescue TemplateParseError => e
+      error "Couldn't parse the `stemcell.json` file: #{e.message}"
+    end
+
+    def launch_instance(role, options={})
+      launcher.run!(role, options)
+    rescue RoleExpansionError
+      error "There was a problem expanding the #{chef_role} role. " \
+            "Perhaps it or one of its dependencies does not exist."
+    rescue MissingStemcellOptionError => e
+      error "The '#{e.option}' attribute needs to be specified on the " \
+            "command line, in the role, in the stemcell.json defaults, " \
+            "or set by the #{e.option.upcase.gsub('-','_')} environment variable."
+    rescue UnknownBackingStoreError => e
+      error "Unknown backing store type: #{e.backing_store}."
+    end
+
+    def tail_converge(instances, options={})
+      puts "\nTailing the initial converge. Press Ctrl-C to exit...".green
+
+      if instances.count > 1
+        puts "\nYou're launching more than one instance."
+        puts "Showing you on the output from #{instances.first.instance_id}."
+      end
+
+      puts "\n"
+      tailer = LogTailer.new(instances.first.public_dns_name, options['ssh_user'])
+      tailer.run!
+    end
+
+    def retrieve_defaults
+      @default_options = launcher ? launcher.default_options : {}
+      @default_branch  = @default_options['git_branch']
+    end
+
+    def configure_option_parser
+      parser_defaults = OPTION_PARSER_DEFAULTS
+      if showing_help
+        parser_defaults = parser_defaults.merge!(@default_options)
+        transform_parser_defaults(parser_defaults)
+      end
+
+      @option_parser = OptionParser.new({
+        :defaults => parser_defaults,
+        :version  => VERSION_STRING,
+        :banner   => BANNER_STRING
+      })
+    end
+
+    def transform_parser_defaults(pd)
+      # There are two special cases: security groups and tags. Security groups
+      # are an array in the template, but trollop expects a string. Likewise,
+      # in the case of tags, they are represented as a hash, but trollop wants
+      # a string once again. In both cases, the data types are encoded as a
+      # comma-separated list when presented as defaults.
+      pd['security_groups'] &&= pd['security_groups'].join(',')
+      pd['tags'] &&= pd['tags'].to_a.map { |p| p.join('=') }.join(',')
+    end
+
+    def parse_options_or_print_help
+      parsed_options = @option_parser.parse!(ARGV)
+      # Parsed options will contain nil values, strip them out.
+      parsed_options.keys.each do |key|
+        value = parsed_options[key]
+        parsed_options.delete(key) unless key.is_a?(String) && value
+      end
+      parsed_options
+    end
+
+    def determine_role(options)
+      role = ARGV.shift
+      @chef_role = role if role && role.length > 0
+      @chef_role ||= options['chef_role']
+    end
+
+    def retrieve_credentials_from_awc(name)
+      creds = AWS::Creds[name || 'default']
+      return creds && {
+        'aws_access_key' => creds.access_key_id,
+        'aws_secret_key' => creds.secret_access_key
+      }
+    rescue AWS::Creds::InvalidKeyTab,
+           AWS::Creds::InvalidKeyPair
+    end
+
+    def validate_chef_root
+      unless chef_root_valid?
+        error "This isn't a chef repository: no roles folder."
+      end
+    end
+
+    def display_chef_repo_warnings
+      # Print a series of warnings about the chef repo state.
+      if not_default_branch?
+        warning "You are not on the '#{@default_branch}' branch!"
+      end
+      if has_unstaged_changes?
+        warning "You have unstaged changes."
+      end
+      if has_uncommitted_changes?
+        warning "Your index contains uncommitted changes."
+      end
+    end
+
+    def chef_root_valid?
+      File.directory?(File.join(chef_root, 'roles'))
+    end
+
+    def not_default_branch?
+      run_in_chef("git rev-parse --abbrev-ref HEAD").strip != @default_branch
+    end
+
+    def has_unstaged_changes?
+      run_in_chef("git diff-files --quiet --ignore-submodules")
+      $? != 0
+    end
+
+    def has_uncommitted_changes?
+      run_in_chef("git diff-index --cached --quiet HEAD --ignore-submodules")
+      $? != 0
+    end
+
+    def run_in_chef(command)
+      `cd #{chef_root}; #{command}`
+    end
+
+    def error(string)
+      warn "\nERROR: #{string}\n".red
+      exit 1
+    end
+
+    def warning(string)
+      warn "\nWARNING: #{string}".red
+    end
+
+    def print_version
+      puts "#{VERSION_STRING}\n"
+    end
+
+    def print_usage_and_exit
+      puts "#{BANNER_STRING}\n"
+      exit
+    end
+  end
+end