bosh_cli_plugin_aws 1.5.0.pre.1113

data/lib/bosh_cli_plugin_aws/bootstrap.rb

@@ -0,0 +1,31 @@
+module Bosh
+  module Aws
+    class Bootstrap
+      AWS_JENKINS_BUCKET = "bosh-jenkins-artifacts"
+
+      attr_accessor :options, :runner
+
+      def initialize(runner, options)
+        self.options = options
+        self.runner = runner
+      end
+
+      def create_user(username, password)
+        user = Bosh::Cli::Command::User.new(runner)
+        user.options = self.options
+        user.create(username, password)
+        login(username, password)
+      end
+
+      def login(username, password)
+        misc = Bosh::Cli::Command::Misc.new(runner)
+        misc.options = self.options
+        misc.login(username, password)
+      end
+
+      def manifest
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/bosh_bootstrap.rb

@@ -0,0 +1,158 @@
+require_relative 'bootstrap'
+require 'net/https'
+require 'bosh/stemcell/archive'
+require 'bosh/stemcell/archive_filename'
+require 'bosh/stemcell/infrastructure'
+require 'bosh/stemcell/operating_system'
+
+module Bosh
+  module Aws
+    BootstrapError = Class.new(StandardError)
+
+    class BoshBootstrap < Bootstrap
+      attr_accessor :director, :s3
+
+      def initialize(director, s3, options)
+        self.options = options
+        self.options[:non_interactive] = true
+        self.director = director
+        self.s3 = s3
+      end
+
+      def validate_requirements
+
+        release_exist = director.list_releases.detect { |r| r['name'] == 'bosh' }
+        first_stemcell = director.list_stemcells.first
+
+        existing_deployments = director.list_deployments.map { |deployment| deployment['name'] }
+
+        if existing_deployments.include? manifest.bosh_deployment_name
+          raise BootstrapError, <<-MSG
+Deployment `#{manifest.bosh_deployment_name}' already exists.
+This command should be used for bootstrapping bosh from scratch.
+          MSG
+        end
+
+        return release_exist, first_stemcell
+      end
+
+      def start
+        release_exist, first_stemcell = validate_requirements
+
+        fetch_and_upload_release unless release_exist
+        if first_stemcell
+          manifest.stemcell_name = first_stemcell['name']
+        else
+          manifest.stemcell_name = Bosh::Stemcell::Archive.new(fetch_and_upload_stemcell).name
+        end
+        generate_manifest
+
+        deploy
+
+        target_bosh_and_log_in
+      end
+
+      private
+
+      def manifest
+        unless @manifest
+          vpc_receipt_filename = File.expand_path('aws_vpc_receipt.yml')
+          route53_receipt_filename = File.expand_path('aws_route53_receipt.yml')
+          bosh_rds_receipt_filename = File.expand_path('aws_rds_bosh_receipt.yml')
+
+          vpc_config = load_yaml_file(vpc_receipt_filename)
+          route53_config = load_yaml_file(route53_receipt_filename)
+          bosh_rds_config = load_yaml_file(bosh_rds_receipt_filename)
+          @manifest = Bosh::Aws::BoshManifest.new(vpc_config, route53_config, director.uuid, bosh_rds_config, options)
+        end
+
+        @manifest
+      end
+
+      def generate_manifest
+        deployment_folder = File.join('deployments', manifest.deployment_name)
+
+        FileUtils.mkdir_p deployment_folder
+        Dir.chdir(deployment_folder) do
+          write_yaml(manifest, manifest.file_name)
+        end
+
+        deployment_command = Bosh::Cli::Command::Deployment.new
+        deployment_command.options = self.options
+        deployment_command.set_current(File.join(deployment_folder, manifest.file_name))
+      end
+
+      def fetch_and_upload_release
+        release_command = Bosh::Cli::Command::Release.new
+        release_command.options = self.options
+        release_command.upload(bosh_release)
+      end
+
+      def target_bosh_and_log_in
+        misc_command = Bosh::Cli::Command::Misc.new
+        misc_command.options = self.options
+        misc_command.set_target(manifest.vip)
+        misc_command.options[:target] = manifest.vip
+        misc_command.login('admin', 'admin')
+
+        self.options[:target] = misc_command.config.target
+      end
+
+      def deploy
+        deployment_command = Bosh::Cli::Command::Deployment.new
+        deployment_command.options = self.options
+        deployment_command.perform
+
+        new_director = Bosh::Cli::Client::Director.new("https://#{manifest.vip}:25555", nil, nil,
+                                               num_retries: 12, retry_wait_interval: 5)
+        new_director.wait_until_ready
+      end
+
+      def fetch_and_upload_stemcell
+        stemcell_command = Bosh::Cli::Command::Stemcell.new
+        stemcell_command.options = self.options
+        stemcell_path = bosh_stemcell
+        stemcell_command.upload(stemcell_path)
+        stemcell_path
+      end
+
+      def bosh_stemcell
+        if bosh_stemcell_override
+          say("Using stemcell #{bosh_stemcell_override}")
+          return bosh_stemcell_override
+        end
+
+        s3.copy_remote_file(AWS_JENKINS_BUCKET,
+                            "bosh-stemcell/aws/#{latest_aws_ubuntu_bosh_stemcell_filename}",
+                            'bosh_stemcell.tgz')
+      end
+
+      def latest_aws_ubuntu_bosh_stemcell_filename
+        infrastructure = Bosh::Stemcell::Infrastructure.for('aws')
+        operating_system = Bosh::Stemcell::OperatingSystem.for('ubuntu')
+        Bosh::Stemcell::ArchiveFilename.new('latest', infrastructure, operating_system, 'bosh-stemcell', true)
+      end
+
+      def bosh_release
+        if bosh_release_override
+          say("Using release #{bosh_release_override}")
+          return bosh_release_override
+        end
+        s3.copy_remote_file(AWS_JENKINS_BUCKET, "release/bosh-#{bosh_version}.tgz", 'bosh_release.tgz')
+      end
+
+      def bosh_stemcell_override
+        ENV['BOSH_OVERRIDE_LIGHT_STEMCELL_URL']
+      end
+
+      def bosh_release_override
+        ENV['BOSH_OVERRIDE_RELEASE_TGZ']
+      end
+
+      def bosh_version
+        ENV['BOSH_VERSION_OVERRIDE'] ||
+            Bosh::Aws::VERSION.split('.').last
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/bosh_manifest.rb

@@ -0,0 +1,71 @@
+module Bosh
+  module Aws
+    class BoshManifest < MicroboshManifest
+
+      attr_reader :director_uuid, :rds_receipt
+
+      attr_accessor :stemcell_name
+
+      def initialize(vpc_receipt, route53_receipt, director_uuid, rds_receipt, options={})
+        super(vpc_receipt, route53_receipt, options)
+        @director_uuid = director_uuid
+        @rds_receipt = rds_receipt
+        @stemcell_name = 'bosh-aws-xen-ubuntu'
+      end
+
+      def file_name
+        "bosh.yml"
+      end
+
+      def deployment_name
+        "bosh"
+      end
+
+      def vip
+        route53_receipt['elastic_ips']['bosh']['ips'][0] || warning('Missing vip field')
+      end
+
+      def bosh_deployment_name
+        "vpc-bosh-#{name}"
+      end
+
+      def director_ssl_key
+        certificate.key.gsub("\n", "\n        ")
+      end
+
+      def director_ssl_cert
+        certificate.certificate.gsub("\n", "\n        ")
+      end
+
+      def bosh_rds_properties
+        rds_receipt['deployment_manifest']['properties']['bosh']
+      end
+
+      def bosh_rds_host
+        bosh_rds_properties['address']
+      end
+
+      def bosh_rds_port
+        bosh_rds_properties['port']
+      end
+
+      def bosh_rds_password
+        bosh_rds_properties['roles'].first['password']
+      end
+
+      def bosh_rds_user
+        bosh_rds_properties['roles'].first['name']
+      end
+
+      # RSpec overloads to_yaml when you set up expectations on an object;
+      # so to_y is just a way to get directly at the to_yaml implementation without fighting RSpec.
+      def to_y
+        ERB.new(File.read(get_template("bosh.yml.erb"))).result(binding)
+      end
+
+      def to_yaml
+        to_y
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/ec2.rb

@@ -0,0 +1,265 @@
+module Bosh
+  module Aws
+    class EC2
+
+      NAT_AMI_ID = {
+        'us-east-1' => 'ami-f619c29f',      # ami-vpc-nat-1.1.0-beta
+        'us-west-1' => 'ami-3bcc9e7e',      # ami-vpc-nat-1.0.0-beta
+        'us-west-2' => 'ami-52ff7262',      # ami-vpc-nat-1.0.0-beta
+        'eu-west-1' => 'ami-e5e2d991',      # ami-vpc-nat-1.1.0-beta
+        'ap-southeast-1' => 'ami-02eb9350', # ami-vpc-nat-1.0.0-beta
+        'ap-northeast-1' => 'ami-14d86d15', # ami-vpc-nat-1.0.0-beta
+        'ap-southeast-2' => 'ami-ab990e91', # ami-vpc-nat-1.0.0-beta
+        'sa-east-1' => 'ami-0039e61d',      # ami-vpc-nat-1.0.0-beta
+      }
+
+      attr_reader :elastic_ips
+
+      def initialize(credentials)
+        @aws_provider = AwsProvider.new(credentials)
+        @elastic_ips = []
+      end
+
+      def instances_count
+        terminatable_instances.size
+      end
+
+      def vpcs
+        aws_ec2.vpcs
+      end
+
+      def dhcp_options
+        aws_ec2.dhcp_options
+      end
+
+      def allocate_elastic_ips(count)
+        count.times do
+          @elastic_ips << allocate_elastic_ip.public_ip
+        end
+        #say "\tallocated #{eip.public_ip}".make_green
+      end
+
+      def allocate_elastic_ip
+        aws_ec2.elastic_ips.allocate(vpc: true)
+      end
+
+      def release_elastic_ips(ips)
+        aws_ec2.elastic_ips.each { |ip| ip.release if ips.include? ip.public_ip }
+      end
+
+      def release_all_elastic_ips
+        releasable_elastic_ips.map(&:release)
+      end
+
+      def create_internet_gateway
+        aws_ec2.internet_gateways.create
+      end
+
+      def internet_gateway_ids
+        aws_ec2.internet_gateways.map(&:id)
+      end
+
+      def delete_internet_gateways(ids)
+        Array(ids).each do |id|
+          gw = aws_ec2.internet_gateways[id]
+          gw.attachments.map(&:delete)
+          gw.delete
+        end
+      end
+
+      def create_instance(options)
+        aws_ec2.instances.create(options)
+      end
+
+      def create_nat_instance(options)
+        name = options["name"]
+        key_pair = select_key_pair_for_instance(name, options["key_name"])
+
+
+        instance_options = {
+            image_id: NAT_AMI_ID[aws_provider.region],
+            instance_type: options.fetch("instance_type", "m1.small"),
+            subnet: options["subnet_id"],
+            private_ip_address: options["ip"],
+            security_groups: [options["security_group"]],
+            key_name: key_pair
+        }
+
+        create_instance(instance_options).tap do |instance|
+          Bosh::AwsCloud::ResourceWait.for_instance(instance: instance, state: :running)
+          instance.add_tag("Name", {value: name})
+          elastic_ip = allocate_elastic_ip
+          Bosh::Common.retryable(tries: 30, on: AWS::EC2::Errors::InvalidAddress::NotFound) do
+            instance.associate_elastic_ip(elastic_ip)
+            true
+          end
+          disable_src_dest_checking(instance.id)
+        end
+      end
+
+      def disable_src_dest_checking(instance_id)
+        aws_ec2.client.modify_instance_attribute(
+            :instance_id => instance_id,
+            :source_dest_check => {:value => false}
+        )
+      end
+
+      def terminate_instances
+        terminatable_instances.each(&:terminate)
+        1.upto(100) do
+          break if terminatable_instances.empty?
+          sleep 4
+        end
+        terminatable_instances.empty?
+      end
+
+      def instance_names
+        terminatable_instances.inject({}) do |memo, instance|
+          memo[instance.instance_id] = instance.tags["Name"] || '<unnamed instance>'
+          memo
+        end
+      end
+
+      def get_running_instance_by_name(name)
+        instances = aws_ec2.instances.select { |instance| instance.tags["Name"] == name && instance.status == :running }
+        raise "More than one running instance with name '#{name}'." if instances.count > 1
+        instances.first
+      end
+
+      def terminatable_instance_names
+        terminatable_instances.inject({}) do |memo, instance|
+          memo[instance.instance_id] = instance.tags["Name"]
+          memo
+        end
+      end
+
+      def delete_volumes
+        unattached_volumes.each do |vol|
+          begin
+            vol.delete
+          rescue AWS::EC2::Errors::InvalidVolume::NotFound
+            # ignored
+          end
+        end
+      end
+
+      def volume_count
+        unattached_volumes.count
+      end
+
+      def add_key_pair(name, path_to_public_private_key)
+        private_key_path = path_to_public_private_key.gsub(/\.pub$/, '')
+        public_key_path = "#{private_key_path}.pub"
+
+        if !File.exist?(private_key_path)
+          system "ssh-keygen", "-q", '-N', "", "-t", "rsa", "-f", private_key_path
+        end
+
+        unless key_pair_by_name(name).nil?
+          err "Key pair #{name} already exists on AWS"
+        end
+
+        aws_ec2.key_pairs.import(name, File.read(public_key_path))
+      end
+
+      def key_pair_by_name(name)
+        key_pairs.detect { |kp| kp.name == name }
+      end
+
+      def key_pairs
+        aws_ec2.key_pairs.to_a
+      end
+
+      def force_add_key_pair(name, path_to_public_private_key)
+        remove_key_pair(name)
+        add_key_pair(name, path_to_public_private_key)
+      end
+
+      def remove_key_pair(name)
+        key_pair = key_pair_by_name(name)
+        key_pair.delete unless key_pair.nil?
+        Bosh::Common.retryable(tries: 15) do
+          key_pair_by_name(name).nil?
+        end
+      end
+
+      def remove_all_key_pairs
+        aws_ec2.key_pairs.each(&:delete)
+
+        Bosh::Common.retryable(tries: 10) do
+          aws_ec2.key_pairs.to_a.empty?
+        end
+      end
+
+      def delete_all_security_groups
+        dsg = deletable_security_groups
+
+        # Revoke all permissions before deleting because a permission can reference
+        # another security group, causing a delete to fail
+        dsg.each do |sg|
+          sg.ingress_ip_permissions.map(&:revoke)
+          sg.egress_ip_permissions.map(&:revoke)
+        end
+
+        dsg.each do |sg|
+          sg.delete unless (sg.name == "default" && !sg.vpc_id)
+        end
+      end
+
+      private
+
+      attr_reader :aws_provider
+
+      def aws_ec2
+        aws_provider.ec2
+      end
+
+      def terminatable_instances
+        aws_ec2.instances.reject do |i|
+          begin
+            i.api_termination_disabled? || i.status.to_s == "terminated"
+          rescue AWS::Core::Resource::NotFound
+            # ignoring instances which disappear while we are going through them
+          end
+        end
+      end
+
+      def releasable_elastic_ips
+        ti = terminatable_instances.map(&:id)
+        aws_ec2.elastic_ips.select { |eip| eip.instance_id.nil? || ti.include?(eip.instance_id) }
+      end
+
+      def deletable_security_groups
+        aws_ec2.security_groups.reject { |sg| security_group_in_use?(sg) }
+      end
+
+      def security_group_in_use?(sg)
+        sg.instances.any? { |s| s.api_termination_disabled? }
+      end
+
+      def unattached_volumes
+        # only check volumes that don't have status 'deleting' and 'deleted'
+        aws_ec2.volumes.filter('status', %w[available creating in_use error]).
+            reject { |v| v.attachments.any? }
+      end
+
+      def select_key_pair_for_instance(name, key_pair)
+        if key_pair.nil?
+          if key_pairs.count > 1
+            raise "AWS key pair name unspecified for instance '#{name}', unable to select a default."
+          elsif key_pairs.count == 0
+            raise "AWS key pair name unspecified for instance '#{name}', no key pairs available to select a default."
+          else
+            key_pair = key_pairs.first.name
+          end
+        else
+          if key_pairs.none? { |kp| kp.name == key_pair }
+            raise "No such key pair '#{key_pair}' on AWS."
+          end
+        end
+
+        key_pair
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/elb.rb

@@ -0,0 +1,132 @@
+module Bosh::Aws
+  class ELB
+    class BadCertificateError < RuntimeError;
+    end
+
+    def initialize(credentials)
+      @aws_provider = AwsProvider.new(credentials)
+    end
+
+    def create(name, vpc, settings, certs)
+      subnet_names = settings['subnets']
+      subnet_ids = vpc.subnets.select { |k, v| subnet_names.include?(k) }.values
+      security_group_name = settings['security_group']
+      security_group_id = vpc.security_group_by_name(security_group_name).id
+      options = {
+        :listeners => [{
+                         port: 80,
+                         protocol: :http,
+                         instance_port: 80,
+                         instance_protocol: :http,
+                       }],
+        :subnets => subnet_ids,
+        :security_groups => [security_group_id]
+      }
+
+      if settings['https']
+        domain = settings['domain']
+        cert_name = settings['ssl_cert']
+        cert = certs[cert_name]
+        dns_record = settings['dns_record']
+
+        certificate = Bosh::Ssl::Certificate.new(cert['private_key_path'],
+                                                 cert['certificate_path'],
+                                                 "#{dns_record}.#{domain}",
+                                                 cert['certificate_chain_path']
+        ).load_or_create
+
+        uploaded_cert = upload_certificate(cert_name, certificate)
+
+        options[:listeners] << {
+          :port => 443,
+          :protocol => :https,
+          :instance_port => 80,
+          :instance_protocol => :http,
+          # passing through 'ssl_certificate_id' is undocumented, but we're
+          # working around a bug filed here: https://github.com/aws/aws-sdk-ruby/issues/216
+          :ssl_certificate_id => uploaded_cert.arn
+        }
+      end
+
+      Bosh::Common.retryable(tries: 15, on: AWS::ELB::Errors::CertificateNotFound) do
+        aws_elb.load_balancers.create(name, options).tap do |new_elb|
+          new_elb.configure_health_check({
+                                           :healthy_threshold => 5,
+                                           :unhealthy_threshold => 2,
+                                           :interval => 5,
+                                           :timeout => 2,
+                                           :target => 'TCP:80'
+                                         })
+        end
+      end
+    end
+
+    def names
+      aws_elb.load_balancers.map(&:name)
+    end
+
+    def server_certificate_names
+      aws_iam.server_certificates.map(&:name)
+    end
+
+    def delete_elbs
+      aws_elb.load_balancers.each(&:delete)
+      delete_server_certificates
+    end
+
+    def delete_server_certificates
+      Bosh::Common.retryable(tries: 5, sleep: 2) do
+        aws_iam.server_certificates.each(&:delete)
+        aws_iam.server_certificates.to_a.empty?
+      end
+    end
+
+    def find_by_name(name)
+      aws_elb.load_balancers.find { |lb| lb.name == name }
+    end
+
+    private
+
+    attr_reader :aws_provider
+
+    def aws_iam
+      aws_provider.iam
+    end
+
+    def aws_elb
+      aws_provider.elb
+    end
+
+    def upload_certificate(name, cert)
+      certificates = aws_iam.server_certificates
+      options = {
+        name: name,
+        certificate_body: cert.certificate,
+        private_key: cert.key
+      }
+
+      options[:certificate_chain] = cert.chain if cert.chain
+
+      begin
+        certificate = nil
+
+        Bosh::Common.retryable(on: AWS::IAM::Errors::MalformedCertificate, tries: 10, sleep: 2) do
+          begin
+            certificate = certificates.upload(options)
+            server_certificate_names.include? name
+          rescue AWS::IAM::Errors::EntityAlreadyExists
+            certificate = aws_iam.server_certificates[name]
+            true
+          end
+        end
+
+        certificate
+      rescue AWS::IAM::Errors::MalformedCertificate => e
+        certificate = cert.certificate
+        private_key = cert.key
+        message = "Certificate:\n#{certificate}\n\nPrivate Key:\n#{private_key}"
+        raise BadCertificateError.new("Unable to upload ELB SSL Certificate: #{e.message}\n#{message}")
+      end
+    end
+  end
+end