stemcell 0.2.9 → 0.4.4

data/lib/stemcell/errors.rb

@@ -0,0 +1,25 @@
+module Stemcell
+  # This is the class from which all stemcell errors descend.
+  class Error < StandardError; end
+
+  class NoTemplateError < Error; end
+  class TemplateParseError < Error; end
+  class RoleExpansionError < Error; end
+  class EmptyRoleError < Error; end
+
+  class UnknownBackingStoreError < Error
+    attr_reader :backing_store
+    def initialize(backing_store)
+      super "Unknown backing store: #{backing_store}"
+      @backing_store = backing_store
+    end
+  end
+
+  class MissingStemcellOptionError < Error
+    attr_reader :option
+    def initialize(option)
+      super "Missing option: #{option}"
+      @option = option
+    end
+  end
+end

data/lib/stemcell/launcher.rb

@@ -0,0 +1,258 @@
+require 'aws-sdk'
+require 'logger'
+require 'erb'
+
+require "stemcell/version"
+require "stemcell/option_parser"
+
+module Stemcell
+  class Launcher
+
+    REQUIRED_OPTIONS = [
+      'aws_access_key',
+      'aws_secret_key',
+      'region'
+    ]
+
+    REQUIRED_LAUNCH_PARAMETERS = [
+      'chef_role',
+      'chef_environment',
+      'chef_data_bag_secret',
+      'git_branch',
+      'git_key',
+      'git_origin',
+      'key_name',
+      'instance_type',
+      'image_id',
+      'security_groups',
+      'availability_zone',
+      'count'
+    ]
+
+    LAUNCH_PARAMETERS = [
+      'chef_role',
+      'chef_environment',
+      'chef_data_bag_secret',
+      'git_branch',
+      'git_key',
+      'git_origin',
+      'key_name',
+      'instance_type',
+      'image_id',
+      'availability_zone',
+      'count',
+      'security_groups',
+      'tags',
+      'iam_role',
+      'ebs_optimized',
+      'block_device_mappings',
+      'ephemeral_devices',
+      'placement_group'
+    ]
+
+    TEMPLATE_PATH = '../templates/bootstrap.sh.erb'
+    LAST_BOOTSTRAP_LINE = "Stemcell bootstrap finished successfully!"
+
+    def initialize(opts={})
+      @log = Logger.new(STDOUT)
+      @log.level = Logger::INFO unless ENV['DEBUG']
+      @log.debug "creating new stemcell object"
+      @log.debug "opts are #{opts.inspect}"
+
+      REQUIRED_OPTIONS.each do |req|
+        raise ArgumentError, "missing required param #{req}" unless opts[req]
+        instance_variable_set("@#{req}",opts[req])
+      end
+
+      @ec2_url = "ec2.#{@region}.amazonaws.com"
+      @timeout = 300
+      @start_time = Time.new
+
+      AWS.config({
+        :access_key_id     => @aws_access_key,
+        :secret_access_key => @aws_secret_key})
+
+      @ec2 = AWS::EC2.new(:ec2_endpoint => @ec2_url)
+    end
+
+
+    def launch(opts={})
+      verify_required_options(opts, REQUIRED_LAUNCH_PARAMETERS)
+
+      # attempt to accept keys as file paths
+      opts['git_key'] = try_file(opts['git_key'])
+      opts['chef_data_bag_secret'] = try_file(opts['chef_data_bag_secret'])
+
+      # generate tags and merge in any that were specefied as inputs
+      tags = {
+        'Name' => "#{opts['chef_role']}-#{opts['chef_environment']}",
+        'Group' => "#{opts['chef_role']}-#{opts['chef_environment']}",
+        'created_by' => ENV['USER'],
+        'stemcell' => VERSION,
+      }
+      tags.merge!(opts['tags']) if opts['tags']
+
+      # generate launch options
+      launch_options = {
+        :image_id => opts['image_id'],
+        :security_groups => opts['security_groups'],
+        :user_data => opts['user_data'],
+        :instance_type => opts['instance_type'],
+        :key_name => opts['key_name'],
+        :count => opts['count'],
+      }
+
+      # specify availability zone (optional)
+      if opts['availability_zone']
+        launch_options[:availability_zone] = opts['availability_zone']
+      end
+
+      # specify IAM role (optional)
+      if opts['iam_role']
+        launch_options[:iam_instance_profile] = opts['iam_role']
+      end
+
+      # specify placement group (optional)
+      if opts['placement_group']
+        launch_options[:placement] = {
+          :group_name => opts['placement_group'],
+        }
+      end
+
+      # specify an EBS-optimized instance (optional)
+      launch_options[:ebs_optimized] = true if opts['ebs_optimized']
+
+      # specify raw block device mappings (optional)
+      if opts['block_device_mappings']
+        launch_options[:block_device_mappings] = opts['block_device_mappings']
+      end
+
+      # specify ephemeral block device mappings (optional)
+      if opts['ephemeral_devices']
+        launch_options[:block_device_mappings] ||= []
+        opts['ephemeral_devices'].each_with_index do |device,i|
+          launch_options[:block_device_mappings].push ({
+            :device_name => device,
+            :virtual_name => "ephemeral#{i}"
+          })
+        end
+      end
+
+      #
+
+      # generate user data script to bootstrap instance, include in launch
+      # options UNLESS we have manually set the user-data (ie. for ec2admin)
+      launch_options[:user_data] ||= render_template(opts)
+
+      # launch instances
+      instances = do_launch(launch_options)
+
+      # wait for aws to report instance stats
+      wait(instances)
+
+      # set tags on all instances launched
+      set_tags(instances, tags)
+
+      print_run_info(instances)
+      @log.info "launched instances successfully"
+      return instances
+    end
+
+    def find_instance(id)
+      return @ec2.instances[id]
+    end
+
+    def kill(instances,opts={})
+      return if instances.nil?
+      instances.each do |i|
+        begin
+          instance = find_instance(i)
+          @log.warn "Terminating instance #{instance.instance_id}"
+          instance.terminate
+        rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
+          throw e unless opts[:ignore_not_found]
+        end
+      end
+    end
+
+    # this is made public for ec2admin usage
+    def render_template(opts={})
+      template_file_path = File.expand_path(TEMPLATE_PATH, __FILE__)
+      template_file = File.read(template_file_path)
+      erb_template = ERB.new(template_file)
+      last_bootstrap_line = LAST_BOOTSTRAP_LINE
+      generated_template = erb_template.result(binding)
+      @log.debug "genereated template is #{generated_template}"
+      return generated_template
+    end
+
+    private
+
+    def print_run_info(instances)
+      puts "\nhere is the info for what's launched:"
+      instances.each do |instance|
+        puts "\tinstance_id: #{instance.instance_id}"
+        puts "\tpublic ip:   #{instance.public_ip_address}"
+        puts
+      end
+      puts "install logs will be in /var/log/init and /var/log/init.err"
+    end
+
+    def wait(instances)
+      @log.info "Waiting up to #{@timeout} seconds for #{instances.count} " \
+                "instance(s) (#{instances.inspect}):"
+
+      while true
+        sleep 5
+        if Time.now - @start_time > @timeout
+          kill(instances)
+          raise TimeoutError, "exceded timeout of #{@timeout}"
+        end
+
+        if instances.select{|i| i.status != :running }.empty?
+          break
+        end
+      end
+
+      @log.info "all instances in running state"
+    end
+
+    def verify_required_options(params,required_options)
+      @log.debug "params is #{params}"
+      @log.debug "required_options are #{required_options}"
+      required_options.each do |required|
+        unless params.include?(required)
+          raise ArgumentError, "you need to provide option #{required}"
+        end
+      end
+    end
+
+    def do_launch(opts={})
+      @log.debug "about to launch instance(s) with options #{opts}"
+      @log.info "launching instances"
+      instances = @ec2.instances.create(opts)
+      instances = [instances] unless Array === instances
+      instances.each do |instance|
+        @log.info "launched instance #{instance.instance_id}"
+      end
+      return instances
+    end
+
+    def set_tags(instances=[],tags)
+      @log.info "setting tags on instance(s)"
+      instances.each do |instance|
+        instance.tags.set(tags)
+      end
+    end
+
+    # attempt to accept keys as file paths
+    def try_file(opt="")
+      begin
+        return File.read(opt)
+      rescue Object => e
+        return opt
+      end
+    end
+
+  end
+end

data/lib/stemcell/log_tailer.rb

@@ -0,0 +1,143 @@
+require 'socket'
+require 'net/ssh'
+
+module Stemcell
+  class LogTailer
+    attr_reader :hostname
+    attr_reader :username
+    attr_reader :ssh_port
+
+    attr_reader :finished
+    attr_reader :interrupted
+
+    # Don't wait more than two minutes
+    MAX_WAIT_FOR_SSH = 120
+
+    TAILING_COMMAND =
+      "while [ ! -s /var/log/init ]; " \
+      "do " \
+        "printf '.' 1>&2; " \
+        "sleep 1; " \
+      "done; " \
+      "echo ' /var/log/init exists!' 1>&2; " \
+      "exec tail -qf /var/log/init*"
+
+    def initialize(hostname, username, ssh_port=22)
+      @hostname = hostname
+      @username = username
+      @ssh_port = ssh_port
+
+      @finished = false
+      @interrupted = false
+    end
+
+    def run!
+      while_catching_interrupt do
+        return unless wait_for_ssh
+        tail_until_interrupt
+      end
+    end
+
+    private
+
+    def wait_for_ssh
+      return if interrupted
+
+      print "Waiting for sshd..."
+
+      start_time = Time.now
+      banner = nil
+
+      loop do
+        print "."
+        begin
+          banner = get_ssh_banner
+          break
+        rescue SocketError,
+               IOError,
+               Errno::ECONNREFUSED,
+               Errno::ECONNRESET,
+               Errno::EHOSTUNREACH,
+               Errno::ENETUNREACH
+          sleep 5
+        rescue Errno::ETIMEDOUT,
+               Errno::EPERM
+        end
+
+        # Don't wait forever
+        if Time.now - start_time > MAX_WAIT_FOR_SSH
+          puts " TIMEOUT!".red
+          return false
+        end
+
+        if done?
+          puts " ABORT!".red
+          return false
+        end
+      end
+
+      puts " UP!".green
+      puts "Server responded with: #{banner.green}"
+      true
+    end
+
+    def tail_until_interrupt
+      return if interrupted
+
+      session = Net::SSH.start(hostname, username)
+
+      channel = session.open_channel do |ch|
+        ch.request_pty do |ch, success|
+          raise "Couldn't start a pseudo-tty!" unless success
+
+          ch.on_data do |ch, data|
+            STDOUT.print(data)
+            @finished = true if contains_last_line?(data)
+          end
+          ch.on_extended_data do |c, type, data|
+            STDERR.print(data)
+          end
+
+          ch.exec(TAILING_COMMAND)
+        end
+      end
+
+      session.loop(1) do
+        if done?
+          # Send an interrupt to kill the remote process
+          channel.send_data(Net::SSH::Connection::Term::VINTR)
+          channel.send_data "exit\n"
+          channel.eof!
+          channel.close
+          false
+        else
+          session.busy?
+        end
+      end
+
+      session.close
+    end
+
+    def done?
+      finished || interrupted
+    end
+
+    def get_ssh_banner
+      socket = TCPSocket.new(hostname, ssh_port)
+      IO.select([socket], nil, nil, 5) ? socket.gets : nil
+    ensure
+      socket.close if socket
+    end
+
+    def contains_last_line?(data)
+      data =~ /#{Launcher::LAST_BOOTSTRAP_LINE}/
+    end
+
+    def while_catching_interrupt
+      trap(:SIGINT) { @interrupted = true }
+      yield
+    ensure
+      trap(:SIGINT, nil) 
+    end
+  end
+end

data/lib/stemcell/metadata_launcher.rb

@@ -0,0 +1,108 @@
+module Stemcell
+  class MetadataLauncher
+    attr_reader :chef_root
+    attr_accessor :interactive
+
+    attr_reader :source
+
+    def initialize(options={})
+      @chef_root = options[:chef_root]
+      @interactive = options.fetch(:interactive, false)
+
+      raise ArgumentError, "You must specify chef_root" unless chef_root
+
+      @source = MetadataSource.new(chef_root)
+    end
+
+    def run!(role, override_options={})
+      environment = expand_environment(override_options)
+      launch_options = determine_options(role, environment, override_options)
+
+      validate_options(launch_options)
+      describe_instance(launch_options)
+      invoke_launcher(launch_options)
+    end
+
+    def default_options
+      source.default_options
+    end
+
+    private
+
+    def expand_environment(override_opts)
+      override_opts['chef_environment'] ||
+        default_options['chef_environment']
+    end
+
+    def determine_options(role, environment, override_options)
+      # Initially assume that empty roles are not allowed
+      allow_empty = false
+      begin
+        return source.expand_role(
+          role,
+          environment,
+          override_options,
+          :allow_empty_roles => allow_empty)
+      rescue EmptyRoleError
+        warn_empty_role
+        allow_empty = true
+        retry
+      end
+    end
+
+    def validate_options(options={})
+      [ Launcher::REQUIRED_OPTIONS,
+        Launcher::REQUIRED_LAUNCH_PARAMETERS
+      ].flatten.each do |arg|
+        if options[arg].nil? or !options[arg]
+          raise Stemcell::MissingStemcellOptionError.new(arg)
+        end
+      end
+    end
+
+    def describe_instance(options={})
+      puts "\nYou're about to launch instance(s) with the following options:\n\n"
+
+      options.keys.sort.each do |key|
+        value = options[key]
+        next unless value
+        spaces = " " * (23 - key.length)
+        puts "  #{key}#{spaces}#{value.to_s.green}"
+      end
+
+      if interactive
+        print "\nProceed? (y/N) "
+        confirm = $stdin.gets
+        exit unless confirm.chomp.downcase == 'y'
+      end
+
+      # One more new line to be pretty.
+      print "\n"
+    end
+
+    def warn_empty_role
+      warn "\nWARNING: This role contains no stemcell attributes.".yellow
+
+      if interactive
+        print "\nDo you want to launch it anyways? (y/N) "
+        confirm = $stdin.gets
+        exit unless confirm.chomp.downcase == 'y'
+      end
+    end
+
+    def invoke_launcher(options={})
+      launcher = Launcher.new({
+        'aws_access_key' => options['aws_access_key'],
+        'aws_secret_key' => options['aws_secret_key'],
+        'region'         => options['region'],
+      })
+      # Slice off just the options used for launching.
+      launch_options = {}
+      Launcher::LAUNCH_PARAMETERS.each do |a|
+        launch_options[a] = options[a]
+      end
+      # Create the instance from these options.
+      launcher.launch(launch_options)
+    end
+  end
+end

data/lib/stemcell/metadata_source.rb

@@ -0,0 +1,139 @@
+require 'chef'
+require 'json'
+
+module Stemcell
+  class MetadataSource
+    attr_reader :chef_root
+    attr_reader :default_options
+
+    DEFAULT_OPTIONS = {
+      'chef_environment' => 'production',
+      'git_branch'       => 'production',
+      'count'            => 1
+    }
+
+    # Search for instance metadata in the following role attributes, with
+    # priority given to the keys at the head.
+    METADATA_ATTRIBUTES = [
+      :instance_metadata,
+      :stemcell
+    ]
+
+    def initialize(chef_root)
+      @chef_root = chef_root
+
+      raise ArgumentError, "You must specify a chef root" unless chef_root
+
+      template_options = read_template
+      @default_options = DEFAULT_OPTIONS.merge(template_options['defaults'])
+
+      @all_backing_store_options = template_options['backing_store']
+      @all_azs_by_region = template_options['availability_zones']
+    end
+
+    def expand_role(role, environment, override_options={}, options={})
+      raise ArgumentError, "Missing chef role" unless role
+      raise ArgumentError, "Missing chef environment" unless environment
+      allow_empty_roles = options.fetch(:allow_empty_roles, false)
+
+      role_options = expand_role_options(role, environment)
+      role_empty   = role_options.nil? || role_options.empty?
+
+      raise EmptyRoleError if !allow_empty_roles && role_empty
+
+      backing_store_options =
+        expand_backing_store_options(
+            default_options,
+            role_options,
+            override_options
+          )
+
+      # Merge all the options together in priority order
+      merged_options = default_options.dup
+      merged_options.deep_merge!(backing_store_options)
+      merged_options.deep_merge!(role_options) if role_options
+      merged_options.deep_merge!(override_options)
+
+      # Add the AZ if not specified
+      if (region = merged_options['region'])
+        merged_options['availability_zone'] ||= random_az_in_region(region)
+      end
+
+      # The chef environment and role used to expand the runlist takes
+      # priority over all other options.
+      merged_options['chef_environment'] = environment
+      merged_options['chef_role']        = role
+
+      merged_options
+    end
+
+    private
+
+    def read_template
+      begin
+        template_path = File.join(chef_root, 'stemcell.json')
+        template_options = JSON.parse(IO.read(template_path))
+      rescue Errno::ENOENT
+        raise NoTemplateError
+      rescue => e
+        raise TemplateParseError, e.message
+      end
+
+      errors = []
+      unless template_options.include?('defaults')
+        errors << 'missing required section "defaults"; should be a hash containing default launch options'
+      end
+
+      if template_options['availability_zones'].nil?
+        errors << 'missing or empty section "availability zones"'
+        errors << '"availability_zones" should be a hash from region name => list of allowed zones in that region'
+      end
+
+      if template_options['backing_store'].nil? or template_options['backing_store'].empty?
+        errors << 'missing or empty section "backing_store"'
+        errors << '"backing_store" should be a hash from store type (like "ebs") => hash of options for that store'
+      end
+
+      unless errors.empty?
+        raise TemplateParseError, errors.join("; ")
+      end
+
+       return template_options
+    end
+
+    def expand_role_options(chef_role, chef_environment)
+      Chef::Config[:role_path] = File.join(chef_root, 'roles')
+      Chef::Config[:data_bag_path] = File.join(chef_root, 'data_bags')
+
+      run_list = Chef::RunList.new
+      run_list << "role[#{chef_role}]"
+
+      expansion = run_list.expand(chef_environment, 'disk')
+      raise RoleExpansionError if expansion.errors?
+
+      default_attrs = expansion.default_attrs
+      override_attrs = expansion.override_attrs
+
+      merged_attrs = default_attrs.merge(override_attrs)
+      METADATA_ATTRIBUTES.inject(nil) { |r, key| r || merged_attrs[key] }
+    end
+
+    def expand_backing_store_options(default_opts, role_opts, override_opts)
+      backing_store   = override_opts['backing_store']
+      backing_store ||= role_opts.to_hash['backing_store'] if role_opts
+      backing_store ||= default_opts['backing_store']
+      backing_store ||= 'instance_store'
+
+      backing_store_options = @all_backing_store_options[backing_store]
+      if backing_store_options.nil?
+        raise Stemcell::UnknownBackingStoreError.new(backing_store)
+      end
+      backing_store_options
+    end
+
+    def random_az_in_region(region)
+      possible_azs = @all_azs_by_region[region] || []
+      possible_azs.sample
+    end
+  end
+end