stemcell 0.2.9 → 0.4.4

data/lib/stemcell/option_parser.rb

@@ -0,0 +1,292 @@
+require 'trollop'
+
+module Stemcell
+  class OptionParser
+    attr_reader :options
+
+    attr_reader :defaults
+    attr_reader :version
+    attr_reader :banner
+    attr_reader :override_help
+
+    OPTION_DEFINITIONS = [
+      {
+        :name  => 'local_chef_root',
+        :desc  => "directory of your local chef repository",
+        :type  => String,
+        :env   =>  'LOCAL_CHEF_ROOT'
+      },
+      {
+        :name  => 'aws_creds',
+        :desc  => "select the aws credentials to use, via the aws-creds gem",
+        :type  => String,
+        :env   =>  'AWS_CREDS'
+      },
+      {
+        :name  => 'aws_access_key',
+        :desc  => "aws access key",
+        :type  => String,
+        :env   => 'AWS_ACCESS_KEY'
+      },
+      {
+        :name  => 'aws_secret_key',
+        :desc  => "aws secret key",
+        :type  => String,
+        :env   => 'AWS_SECRET_KEY'
+      },
+      {
+        :name  => 'region',
+        :desc  => "ec2 region to launch in",
+        :type  => String,
+        :env   => 'REGION'
+      },
+      {
+        :name  => 'instance_type',
+        :desc  => "machine type to launch",
+        :type  => String,
+        :env   => 'INSTANCE_TYPE'
+      },
+      {
+        :name  => 'backing_store',
+        :desc  => "select between the backing store templates",
+        :type  => String,
+        :env   =>  'BACKING_STORE'
+      },
+      {
+        :name  => 'image_id',
+        :desc  => "ami to use for launch",
+        :type  => String,
+        :env   => 'IMAGE_ID'
+      },
+      {
+        :name  => 'security_groups',
+        :desc  => "comma-separated list of security groups to launch instance with",
+        :type  => String,
+        :env   => 'SECURITY_GROUPS'
+      },
+      {
+        :name  => 'availability_zone',
+        :desc  => "zone in which to launch instances",
+        :type  => String,
+        :env   => 'AVAILABILITY_ZONE'
+      },
+      {
+        :name  => 'tags',
+        :desc  => "comma-separated list of key=value pairs to apply",
+        :type  => String,
+        :env   => 'TAGS'
+      },
+      {
+        :name  => 'key_name',
+        :desc  => "aws ssh key name for the ubuntu user",
+        :type  => String,
+        :env   => 'KEY_NAME'
+      },
+      {
+        :name  => 'iam_role',
+        :desc  => "IAM role to associate with the instance",
+        :type  => String,
+        :env   => 'IAM_ROLE'
+      },
+      {
+        :name  => 'placement_group',
+        :desc  => "Placement group to associate with the instance",
+        :type  => String,
+        :env   => 'PLACEMENT_GROUP'
+      },
+      {
+        :name  => 'ebs_optimized',
+        :desc  => "launch an EBS-Optimized instance",
+        :type  => String,
+        :env   => 'EBS_OPTIMIZED'
+      },
+      {
+        :name  => 'block_device_mappings',
+        :desc  => 'block device mappings',
+        :type  => String,
+        :env   => 'BLOCK_DEVICE_MAPPINGS'
+      },
+      {
+        :name  => 'ephemeral_devices',
+        :desc  => "comma-separated list of block devices to map ephemeral devices to",
+        :type  => String,
+        :env   => 'EPHEMERAL_DEVICES'
+      },
+      {
+        :name  => 'chef_data_bag_secret',
+        :desc  => "path to secret file (or the string containing the secret)",
+        :type  => String,
+        :env   => 'CHEF_DATA_BAG_SECRET'
+      },
+      {
+        :name => 'chef_role',
+        :desc => "chef role of instance to be launched",
+        :type => String,
+        :env  => 'CHEF_ROLE'
+      },
+      {
+        :name  => 'chef_environment',
+        :desc  => "chef environment in which this instance will run",
+        :type  => String,
+        :env   => 'CHEF_ENVIRONMENT'
+      },
+      {
+        :name  => 'git_origin',
+        :desc  => "git origin to use",
+        :type  => String,
+        :env   => 'GIT_ORIGIN'
+      },
+      {
+        :name  => 'git_branch',
+        :desc  => "git branch to run off",
+        :type  => String,
+        :env   => 'GIT_BRANCH'
+      },
+      {
+        :name  => 'git_key',
+        :desc  => "path to the git repo deploy key (or the key as a string)",
+        :type  => String,
+        :env   => 'GIT_KEY'
+      },
+      {
+        :name  => 'count',
+        :desc  => "number of instances to launch",
+        :type  => Integer,
+        :env   => 'COUNT'
+      },
+      {
+        :name  => 'tail',
+        :desc  => "interactively tail the initial converge",
+        :type  => nil,
+        :env   => 'TAIL',
+        :short => :t
+      },
+      {
+        :name  => 'ssh_user',
+        :desc  => "ssh username",
+        :type  => String,
+        :env   => 'SSH_USER',
+        :short => :u
+      },
+      {
+        :name  => 'non_interactive',
+        :desc  => "assumes an affirmative answer to all prompts",
+        :type  => nil,
+        :env   => 'NON_INTERACTIVE',
+        :short => :f
+      }
+    ]
+
+    def initialize(config={})
+      @defaults = config[:defaults] || {}
+      @version = config[:version]
+      @banner = config[:banner]
+      @override_help = config[:override_help]
+    end
+
+    def parse!(args)
+      # The block passed to Trollop#options is evaluated in the binding of the
+      # trollop parser itself, it doesn't have access to the this instance.
+      # So use a value that can be captured instead!
+      _this = self
+      _defns = OPTION_DEFINITIONS
+
+      @options = Trollop::options(args) do
+        version _this.version if _this.version
+        banner  _this.banner  if _this.banner
+
+        _defns.each do |defn|
+          # Prioritize the environment variable, then the given default
+          default = ENV[defn[:env]] || _this.defaults[defn[:name]]
+
+          opt(
+            defn[:name],
+            defn[:desc],
+              :type    => defn[:type],
+              :short   => defn[:short],
+              :default => default)
+        end
+
+        # Prevent trollop from showing its help screen
+        opt('help', 'help', :short => :l) if _this.override_help
+      end
+
+      # convert tags from string to ruby hash
+      if options['tags']
+        tags = {}
+        options['tags'].split(',').each do |tag_set|
+          key, value = tag_set.split('=')
+          tags[key] = value
+        end
+        options['tags'] = tags
+      end
+
+      # parse block_device_mappings to convert it from the standard CLI format
+      # to the EC2 Ruby API format.
+      # All of this is a bit hard to find so here are some docs links to
+      # understand
+
+      # CLI This format is documented by typing
+      # ec2-run-instances --help and looking at the -b option
+      # Basically, it's either
+
+      # none
+      # ephemeral<number>
+      # '[<snapshot-id>][:<size>[:<delete-on-termination>][:<type>[:<iops>]]]'
+
+      # Ruby API (that does call to the native API)
+      # gems/aws-sdk-1.17.0/lib/aws/ec2/instance_collection.rb
+      # line 91 + example line 57
+
+      if options['block_device_mappings']
+        block_device_mappings = []
+        options['block_device_mappings'].split(',').each do |device_set|
+          device,devparam = device_set.split('=')
+
+          mapping = {}
+
+          if devparam == 'none'
+            mapping = { :no_device => device }
+          else
+            mapping = { :device_name => device }
+            if devparam =~ /^ephemeral[0-3]/
+              mapping[:virtual_name] = devparam
+            else
+              # we have a more complex 'ebs' parameter
+              #'[<snapshot-id>][:<size>[:<delete-on-termination>][:<type>[:<iops>]]]'
+
+              mapping[:ebs] = {}
+
+              devparam = devparam.split ':'
+
+              # a bit ugly but short and won't change
+              # notice the to_i on volume_size parameter
+              mapping[:ebs][:snapshot_id] = devparam[0] unless devparam[0].blank?
+              mapping[:ebs][:volume_size] = devparam[1].to_i
+
+              # defaults to true - except if we have the exact string "false"
+              mapping[:ebs][:delete_on_termination] = (devparam[2] != "false")
+
+              # optional. notice the to_i on iops parameter
+              mapping[:ebs][:volume_type] = devparam[3] unless devparam[3].blank?
+              mapping[:ebs][:iops] = devparam[4].to_i if (devparam[4].to_i)
+
+            end
+          end
+
+          block_device_mappings.push mapping
+        end
+
+        options['block_device_mappings'] = block_device_mappings
+      end
+
+      # convert security_groups from comma seperated string to ruby array
+      options['security_groups'] &&= options['security_groups'].split(',')
+      # convert ephemeral_devices from comma separated string to ruby array
+      options['ephemeral_devices'] &&= options['ephemeral_devices'].split(',')
+
+      options
+    end
+
+  end
+end

data/lib/stemcell/templates/bootstrap.sh.erb

@@ -9,6 +9,7 @@
 
 
 set -o pipefail
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 
 # ensure we were called by root
@@ -17,6 +18,29 @@ if [ $UID != 0 ]; then
   exit 1
 fi
 
+## Only run once at a time
+# http://stackoverflow.com/questions/959475/shell-fragment-to-make-sure-only-one-instance-a-shell-script-runs-at-any-given-t
+# slightly modified to use $$
+# to run it I must set +e
+
+SCRIPTNAME=`basename $0`
+PIDFILE=/var/run/${SCRIPTNAME}.pid
+
+set +e
+if [ -f ${PIDFILE} ]; then
+   #verify if the process is actually still running under this pid
+   OLDPID=`cat ${PIDFILE}`
+   RESULT=`ps -ef | grep ${OLDPID} | grep ${SCRIPTNAME}`
+
+   if [ -n "${RESULT}" ]; then
+     echo "Script already running! Exiting"
+     exit 255
+   fi
+
+fi
+### ---
+set -e
+
 
 # redirect stdout to /var/log/init
 exec > /var/log/init
@@ -24,6 +48,10 @@ exec > /var/log/init
 # redirect stderr to /var/log/init.err
 exec 2> /var/log/init.err
 
+#grab pid of this process and update the pid file with it
+echo $$ > ${PIDFILE}
+
+
 
 ##
 ## settings
@@ -32,7 +60,7 @@ exec 2> /var/log/init.err
 
 chef_version=11.4.0
 chef_dir=/etc/chef
-converge=/usr/local/bin/converge
+converge=/usr/local/bin/first_converge
 role=<%= opts['chef_role'] %>
 environment=<%= opts['chef_environment'] %>
 origin=<%= opts['git_origin'] %>
@@ -118,7 +146,10 @@ log_level        :info
 log_location     STDOUT
 EOF
 
-  echo -e "$data_bag_secret" > ${chef_dir}/encrypted_data_bag_secret
+  encrypted_data_bag_secret_file=${chef_dir}/encrypted_data_bag_secret
+  echo -e "$data_bag_secret" > $encrypted_data_bag_secret_file
+  chmod 0400 $encrypted_data_bag_secret_file
+
   echo "chef configured"
 }
 
@@ -213,4 +244,10 @@ configure_chef_daemon
 ##
 
 
-echo "ran successfully:)"
+if [ -f ${PIDFILE} ]; then
+    rm ${PIDFILE}
+fi
+# I can delete the cron too
+rm -f /etc/cron.d/ec2admin_get_callback1
+
+echo "<%= last_bootstrap_line %>"

data/lib/stemcell/utility/deep_merge.rb

@@ -0,0 +1,13 @@
+class Hash
+  def deep_merge!(other_hash)
+    merge!(other_hash) do |key, oldval, newval|
+      # Coerce Chef-style attribute hashes to generic ones
+      oldval = oldval.to_hash if oldval.respond_to?(:to_hash)
+      newval = newval.to_hash if newval.respond_to?(:to_hash)
+
+      oldval.class.to_s == 'Hash' &&
+        newval.class.to_s == 'Hash' ?
+          oldval.deep_merge!(newval) : newval
+    end
+  end
+end

data/lib/stemcell/utility.rb

@@ -0,0 +1 @@
+require 'stemcell/utility/deep_merge'

data/lib/stemcell/version.rb

@@ -1,3 +1,3 @@
 module Stemcell
-  VERSION = "0.2.9"
+  VERSION = "0.4.4"
 end

data/lib/stemcell.rb

@@ -1,189 +1,10 @@
-require 'logger'
-require 'erb'
-require 'aws-sdk'
+require 'stemcell/utility'
+require 'stemcell/errors'
 
-require_relative "stemcell/version"
+require 'stemcell/command_line'
+require 'stemcell/option_parser'
+require 'stemcell/launcher'
+require 'stemcell/log_tailer'
 
-module Stemcell
-  class Stemcell
-    def initialize(opts={})
-      @log = Logger.new(STDOUT)
-      @log.level = Logger::INFO unless ENV['DEBUG']
-      @log.debug "creating new stemcell object"
-      @log.debug "opts are #{opts.inspect}"
-      ['aws_access_key',
-       'aws_secret_key',
-       'region',
-      ].each do |req|
-        raise ArgumentError, "missing required param #{req}" unless opts[req]
-        instance_variable_set("@#{req}",opts[req])
-      end
-
-      @ec2_url = "ec2.#{@region}.amazonaws.com"
-      @timeout = 120
-      @start_time = Time.new
-
-      AWS.config({:access_key_id => @aws_access_key, :secret_access_key => @aws_secret_key})
-      @ec2 = AWS::EC2.new(:ec2_endpoint => @ec2_url)
-    end
-
-
-    def launch(opts={})
-      verify_required_options(opts,[
-        'image_id',
-        'security_groups',
-        'key_name',
-        'count',
-        'chef_role',
-        'chef_environment',
-        'chef_data_bag_secret',
-        'git_branch',
-        'git_key',
-        'git_origin',
-        'instance_type',
-      ])
-
-      # attempt to accept keys as file paths
-      opts['git_key'] = try_file(opts['git_key'])
-      opts['chef_data_bag_secret'] = try_file(opts['chef_data_bag_secret'])
-
-      # generate tags and merge in any that were specefied as inputs
-      tags = {
-        'Name' => "#{opts['chef_role']}-#{opts['chef_environment']}",
-        'Group' => "#{opts['chef_role']}-#{opts['chef_environment']}",
-        'created_by' => ENV['USER'],
-        'stemcell' => VERSION,
-      }
-      tags.merge!(opts['tags']) if opts['tags']
-
-      # generate launch options
-      launch_options = {
-        :image_id => opts['image_id'],
-        :security_groups => opts['security_groups'],
-        :user_data => opts['user_data'],
-        :instance_type => opts['instance_type'],
-        :key_name => opts['key_name'],
-        :count => opts['count'],
-      }
-
-      # specify availability zone (optional)
-      launch_options[:availability_zone] = opts['availability_zone'] if opts['availability_zone']
-
-      # specify IAM role (optional)
-      launch_options[:iam_instance_profile] = opts['iam_role'] if opts['iam_role']
-
-      # specify an EBS-optimized instance (optional)
-      launch_options[:ebs_optimized] = true if opts['ebs_optimized']
-
-      # generate user data script to bootstrap instance, include in launch optsions
-      launch_options[:user_data] = render_template(opts)
-
-      # launch instances
-      instances = do_launch(launch_options)
-
-      # wait for aws to report instance stats
-      wait(instances)
-
-      # set tags on all instances launched
-      set_tags(instances, tags)
-
-      print_run_info(instances)
-      @log.info "launched instances successfully"
-      return instances
-    end
-
-    def find_instance(id)
-      return @ec2.instances[id]
-    end
-
-    def kill(instances,opts={})
-      return if instances.nil?
-      instances.each do |i|
-        begin
-          instance = find_instance(i)
-          @log.warn "Terminating instance #{instance.instance_id}"
-          instance.terminate
-        rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
-          throw e unless opts[:ignore_not_found]
-        end
-      end
-    end
-
-    private
-
-    def print_run_info(instances)
-      puts "here is the info for what's launched:"
-      instances.each do |instance|
-        puts "\tinstance_id: #{instance.instance_id}"
-        puts "\tpublic ip:   #{instance.public_ip_address}"
-        puts
-      end
-      puts "install logs will be in /var/log/init and /var/log/init.err"
-    end
-
-    def wait(instances)
-      @log.info "Waiting up to #{@timeout} seconds for #{instances.count} instances (#{instances.inspect}):"
-
-      while true
-        sleep 5
-        if Time.now - @start_time > @timeout
-          kill(instances)
-          raise TimeoutError, "exceded timeout of #{@timeout}"
-        end
-
-        if instances.select{|i| i.status != :running }.empty?
-          break
-        end
-      end
-
-      @log.info "all instances in running state"
-    end
-
-    def verify_required_options(params,required_options)
-      @log.debug "params is #{params}"
-      @log.debug "required_options are #{required_options}"
-      required_options.each do |required|
-        raise ArgumentError, "you need to provide option #{required}" unless params.include?(required)
-      end
-    end
-
-    def do_launch(opts={})
-      @log.debug "about to launch instance(s) with options #{opts}"
-      @log.info "launching instances"
-      instances = @ec2.instances.create(opts)
-      instances = [instances] unless instances.class == Array
-      instances.each do |instance|
-        @log.info "launched instance #{instance.instance_id}"
-      end
-      return instances
-    end
-
-    def set_tags(instances=[],tags)
-      @log.info "setting tags on instance(s)"
-      instances.each do |instance|
-        instance.tags.set(tags)
-      end
-    end
-
-    def render_template(opts={})
-      this_file = File.expand_path __FILE__
-      base_dir = File.dirname this_file
-      template_file_path = File.join(base_dir,'stemcell','templates','bootstrap.sh.erb')
-      template_file = File.read(template_file_path)
-      erb_template = ERB.new(template_file)
-      generated_template = erb_template.result(binding)
-      @log.debug "genereated template is #{generated_template}"
-      return generated_template
-    end
-
-    # attempt to accept keys as file paths
-    def try_file(opt="")
-      begin
-        return File.read(opt)
-      rescue Object => e
-        return opt
-      end
-    end
-
-  end
-end
+require 'stemcell/metadata_launcher'
+require 'stemcell/metadata_source'

data/stemcell.gemspec

@@ -1,22 +1,27 @@
 # -*- encoding: utf-8 -*-
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+$:.push File.expand_path("../lib", __FILE__)
 require 'stemcell/version'
 
-Gem::Specification.new do |gem|
-  gem.name          = "stemcell"
-  gem.version       = Stemcell::VERSION
-  gem.authors       = ["Martin Rhoads"]
-  gem.email         = ["martin.rhoads@airbnb.com"]
-  gem.description   = %q{stemcell launches instances}
-  gem.summary       = %q{no summary}
-  gem.homepage      = ""
+Gem::Specification.new do |s|
+  s.name          = "stemcell"
+  s.version       = Stemcell::VERSION
+  s.authors       = ["Martin Rhoads", "Igor Serebryany", "Nelson Gauthier", "Patrick Viet"]
+  s.email         = ["martin.rhoads@airbnb.com", "igor.serebryany@airbnb.com"]
+  s.description   = %q{A tool for launching and bootstrapping EC2 instances}
+  s.summary       = %q{no summary}
+  s.homepage      = "https://github.com/airbnb/stemcell"
 
-  gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.require_paths = ["lib"]
-  gem.add_runtime_dependency 'trollop'
-  gem.add_runtime_dependency 'aws-sdk'
-end
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency 'aws-sdk',   '~> 1.9'
+  s.add_runtime_dependency 'net-ssh',   '~> 2.6'
+  s.add_runtime_dependency 'chef',      '~> 11.4.0'
 
+  s.add_runtime_dependency 'trollop',   '~> 2.0'
+  s.add_runtime_dependency 'aws-creds', '~> 0.2.2'
+  s.add_runtime_dependency 'colored',   '~> 1.2'
+  s.add_runtime_dependency 'json',      '~> 1.7.7'
+end