bosh_cli_plugin_aws 1.5.0.pre.1113

data/lib/bosh_cli_plugin_aws/micro_bosh_bootstrap.rb

@@ -0,0 +1,64 @@
+require_relative "bootstrap"
+
+module Bosh
+  module Aws
+    class MicroBoshBootstrap < Bootstrap
+      def start
+        cleanup_previous_deployments
+        generate_deployment_manifest
+        deploy
+
+        login("admin", "admin")
+      end
+
+      def deploy
+        Dir.chdir("deployments") do
+          micro = Bosh::Cli::Command::Micro.new(runner)
+          micro.options = self.options
+          micro.micro_deployment("micro")
+          micro.perform(micro_ami)
+        end
+      end
+
+      def manifest
+        unless @manifest
+          vpc_receipt_filename = File.expand_path("aws_vpc_receipt.yml")
+          route53_receipt_filename = File.expand_path("aws_route53_receipt.yml")
+
+          vpc_config = load_yaml_file(vpc_receipt_filename)
+          route53_config = load_yaml_file(route53_receipt_filename)
+
+          @manifest = Bosh::Aws::MicroboshManifest.new(vpc_config, route53_config, options)
+        end
+
+        @manifest
+      end
+
+      def generate_deployment_manifest
+        deployment_folder = File.join("deployments", manifest.deployment_name)
+
+        FileUtils.mkdir_p deployment_folder
+        if File.exists?(manifest.certificate.certificate_path)
+          FileUtils.cp manifest.certificate.certificate_path, File.join(deployment_folder, manifest.certificate.certificate_path)
+        end
+        if File.exists?(manifest.certificate.key_path)
+          FileUtils.cp manifest.certificate.key_path, File.join(deployment_folder, manifest.certificate.key_path)
+        end
+
+        Dir.chdir(deployment_folder) do
+          write_yaml(manifest, manifest.file_name)
+        end
+      end
+
+      def cleanup_previous_deployments
+        rm_files = %w[bosh-deployments.yml micro bosh-registry.log]
+        rm_files.each { |file| FileUtils.rm_rf File.join("deployments", file) }
+      end
+
+      def micro_ami
+        ENV["BOSH_OVERRIDE_MICRO_STEMCELL_AMI"] ||
+            Net::HTTP.get("#{AWS_JENKINS_BUCKET}.s3.amazonaws.com", "/last_successful-bosh-stemcell-aws_ami_us-east-1").strip
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/microbosh_manifest.rb

@@ -0,0 +1,117 @@
+require 'common/ssl'
+
+module Bosh
+  module Aws
+    class MicroboshManifest
+      attr_reader :vpc_receipt, :route53_receipt, :hm_director_password, :hm_director_user
+
+      def initialize(vpc_receipt, route53_receipt, options={})
+        @vpc_receipt = vpc_receipt
+        @route53_receipt = route53_receipt
+        @hm_director_user = options[:hm_director_user]
+        @hm_director_password = options[:hm_director_password]
+      end
+
+      def file_name
+        "micro_bosh.yml"
+      end
+
+      def deployment_name
+        "micro"
+      end
+
+      def vpc_config
+        vpc_receipt['original_configuration']
+      end
+
+      def name
+        vpc_config['name'] || warning('Missing name field')
+      end
+
+      def vip
+        route53_receipt['elastic_ips']['micro']['ips'][0] || warning('Missing vip field')
+      end
+
+      def subnet
+        vpc_receipt['vpc']['subnets']['bosh1'] || warning('Missing bosh subnet field')
+      end
+
+      def availability_zone
+        vpc_config['vpc']['subnets']['bosh1']['availability_zone'] || warning('Missing availability zone in vpc.subnets.bosh')
+      end
+
+      def access_key_id
+        vpc_config['aws']['access_key_id'] || warning('Missing aws access_key_id field')
+      end
+
+      def secret_access_key
+        vpc_config['aws']['secret_access_key'] || warning('Missing aws secret_access_key field')
+      end
+
+      def region
+        vpc_config['aws']['region'] || warning('Missing aws region field')
+      end
+
+      def key_pair_name
+        vpc_config['key_pairs'].any? ? vpc_config['key_pairs'].keys[0] : warning("Missing key_pairs field, must have at least 1 keypair")
+      end
+
+      def private_key_path
+        vpc_config['key_pairs'].any? ? vpc_config['key_pairs'].values[0].gsub(/\.pub$/, '') : warning("Missing key_pairs field, must have at least 1 keypair")
+      end
+
+      def compiled_package_cache?
+        !!vpc_config['compiled_package_cache']
+      end
+
+      def cache_access_key_id
+        vpc_config['compiled_package_cache']['access_key_id'] || warning('Missing compiled_package_cache access_key_id field')
+      end
+
+      def cache_secret_access_key
+        vpc_config['compiled_package_cache']['secret_access_key'] || warning('Missing compiled_package_cache secret_access_key field')
+      end
+
+      def cache_bucket_name
+        vpc_config['compiled_package_cache']['bucket_name'] || warning('Missing compiled_package_cache bucket_name field')
+      end
+
+      def director_ssl_key
+        certificate.key.gsub("\n", "\n          ")
+      end
+
+      def director_ssl_cert
+        certificate.certificate.gsub("\n", "\n          ")
+      end
+
+      def certificate
+        @cert if @cert
+        key_path = director_ssl['private_key_path'] || 'director.key'
+        cert_path = director_ssl['certificate_path'] || 'director.pem'
+        @cert = Bosh::Ssl::Certificate.new(key_path, cert_path, "*.#{vpc_config['vpc']['domain']}").load_or_create
+      end
+
+      def director_ssl
+        ssl_certs['director_cert'] || {}
+      end
+
+      def ssl_certs
+        vpc_config['ssl_certs'] || {}
+      end
+
+      # RSpec overloads to_yaml when you set up expectations on an object;
+      # so to_y is just a way to get directly at the to_yaml implementation without fighting RSpec.
+      def to_y
+        ERB.new(File.read(get_template("micro_bosh.yml.erb"))).result(binding)
+      end
+
+      def to_yaml
+        to_y
+      end
+
+      def get_template(template)
+        File.expand_path("../../../templates/#{template}", __FILE__)
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/migration.rb

@@ -0,0 +1,40 @@
+module Bosh
+  module Aws
+    class Migration
+
+      attr_reader :s3, :elb, :ec2, :rds, :route53, :logger, :config
+
+      def initialize(config, receipt_bucket_name)
+        @config = config
+        @receipt_bucket_name = receipt_bucket_name
+        aws_config = config['aws']
+        @s3 = S3.new(aws_config)
+        @elb = ELB.new(aws_config)
+        @ec2 = EC2.new(aws_config)
+        @rds = RDS.new(aws_config)
+        @route53 = Route53.new(aws_config)
+        @logger = Bosh::Clouds::Config.logger
+      end
+
+      def run
+        say "Executing migration #{self.class.name}"
+        execute
+      end
+
+      def save_receipt(receipt_name, receipt)
+        receipt_yaml = YAML.dump(receipt)
+        s3.upload_to_bucket(@receipt_bucket_name, "receipts/#{receipt_name}.yml", receipt_yaml)
+
+        File.open("#{receipt_name}.yml", "w+") do |f|
+          f.write(receipt_yaml)
+        end
+
+        say "details in S3 receipt: #{receipt_name} and file: #{receipt_name}.yml"
+      end
+
+      def load_receipt(receipt_name)
+        YAML.load(s3.fetch_object_contents(@receipt_bucket_name, "receipts/#{receipt_name}.yml"))
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/migration_helper.rb

@@ -0,0 +1,150 @@
+module Bosh
+  module Aws
+    module MigrationHelper
+      class Template
+        attr_reader :timestamp_string, :name, :class_name
+
+        def initialize(name)
+          @timestamp_string = Time.new.getutc.strftime("%Y%m%d%H%M%S")
+          @name = name
+          @class_name = MigrationHelper.to_class_name(name)
+        end
+
+        def file_prefix
+          "#{timestamp_string}_#{name}"
+        end
+
+        def render(template_name = "aws_migration")
+          template_file_path = File.expand_path("../../templates/#{template_name}.erb", File.dirname(__FILE__))
+          template = ERB.new(File.new(template_file_path).read(), 0, '<>%-')
+          template.result(binding)
+        end
+      end
+
+      class RdsDb
+
+        attr_accessor :instance_id, :receipt, :tag, :subnet_ids
+
+        def initialize(args = {})
+          vpc_receipt   = args.fetch(:vpc_receipt).fetch('vpc')
+          vpc_subnets   = vpc_receipt.fetch('subnets')
+          @rds_db_conf  = args.fetch(:rds_db_conf)
+          @instance_id  = @rds_db_conf.fetch('instance')
+          @tag          = @rds_db_conf.fetch('tag')
+          @subnet_ids   = @rds_db_conf.fetch('subnets').map { |s| vpc_subnets[s] }
+          @vpc_id       = vpc_receipt.fetch('id')
+        end
+
+        def create!
+          return if RdsDb.aws_rds.database_exists? @instance_id
+          creation_opts = [@instance_id, @subnet_ids, @vpc_id]
+
+          if @rds_db_conf["aws_creation_options"]
+            creation_opts << @rds_db_conf["aws_creation_options"]
+          end
+
+          @response = RdsDb.aws_rds.create_database(*creation_opts)
+          output_rds_properties
+        end
+
+        def output_rds_properties
+          RdsDb.receipt["deployment_manifest"] ||= {}
+          RdsDb.receipt["deployment_manifest"]["properties"] ||= {}
+          RdsDb.receipt["deployment_manifest"]["properties"][@instance_id] = {
+              "db_scheme" => @response[:engine],
+              "roles" => [
+                  {
+                      "tag" => "admin",
+                      "name" => @response[:master_username],
+                      "password" => @response[:master_user_password]
+                  }
+              ],
+              "databases" => [
+                  {
+                      "tag" => @tag,
+                      "name" => @instance_id
+                  }
+              ]
+          }
+        end
+
+        def update_receipt
+          return unless RdsDb.deployment_properties[instance_id]
+
+          db_instance = RdsDb.aws_rds.database(instance_id)
+          RdsDb.receipt['deployment_manifest']['properties'][instance_id].merge!(
+               'address' => db_instance.endpoint_address,
+               'port'    => db_instance.endpoint_port
+          )
+        end
+
+        def self.deployment_properties
+          RdsDb.receipt.fetch('deployment_manifest', {}).fetch('properties', {})
+        end
+
+        def self.aws_rds
+          @aws_rds
+        end
+
+        def self.aws_rds=(arg)
+          @aws_rds = arg
+        end
+
+        def self.receipt
+          @receipt ||= {}
+        end
+
+        def self.was_rds_eventually_available?
+          return true if all_rds_instances_available?(:silent => true)
+          (1..540).any? do |attempt|  # wait up to 3 hours, checking every 20s
+            Kernel.sleep 20
+            all_rds_instances_available?
+          end
+        end
+
+        def self.all_rds_instances_available?(opts = {})
+          silent = opts[:silent]
+          say("checking rds status...") unless silent
+          aws_rds.databases.all? do |db_instance|
+            say("  #{db_instance.db_name} #{db_instance.db_instance_status} #{db_instance.endpoint_address}") unless silent
+            !db_instance.endpoint_address.nil?
+          end
+        end
+      end
+
+
+      def self.aws_migration_directory
+        File.expand_path("../../migrations", File.dirname(__FILE__))
+      end
+
+      def self.aws_spec_migration_directory
+        File.expand_path("../../spec/migrations", File.dirname(__FILE__))
+      end
+
+      def self.generate_migration_file(name)
+        template = Template.new(name)
+
+        filename = "#{aws_migration_directory}/#{template.file_prefix}.rb"
+        spec_filename = "#{aws_spec_migration_directory}/#{template.file_prefix}_spec.rb"
+
+        puts "Creating #{filename} and #{spec_filename}"
+
+        File.open(filename, 'w+') { |f| f.write(template.render) }
+        File.open(spec_filename, 'w+') { |f| f.write(template.render("aws_migration_spec")) }
+      end
+
+      def self.to_class_name(name)
+        name.split('_').map(&:capitalize).join('')
+      end
+
+      def self.all_rds_instances_available?(rds, opts = {})
+        silent = opts[:silent]
+        say("checking rds status...") unless silent
+        rds.databases.all? do |db_instance|
+          say("  #{db_instance.db_name} #{db_instance.db_instance_status} #{db_instance.endpoint_address}") unless silent
+          !db_instance.endpoint_address.nil?
+        end
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/migrator.rb

@@ -0,0 +1,137 @@
+module Bosh
+  module Aws
+    class Migrator
+
+      def initialize(config)
+        @config = config
+        @migration_path = MigrationHelper.aws_migration_directory
+      end
+
+      def migrate
+        return unless needs_migration?
+
+        run_migrations(pending_migrations)
+      end
+
+      def migrate_version(version)
+        return unless needs_migration?
+
+        migration_to_run = pending_migrations.detect do |migration|
+          migration.version == version.to_i
+        end
+
+        run_migrations([migration_to_run])
+      end
+
+      def pending_migrations
+        migrations - environment_migrations
+      end
+
+      def migrations
+        @migrations ||= load_migrations
+      end
+
+      def environment_migrations
+        @environment_migrations ||= load_migrations_for_env
+      end
+
+      def needs_migration?
+        ensure_bucket_exists
+        environment_migrations.nil? || environment_migrations != migrations
+      end
+
+      private
+
+      attr_reader :migration_path
+
+      def aws_s3
+        @aws_s3 ||= Bosh::Aws::S3.new(@config['aws'])
+      end
+
+      def ensure_bucket_exists
+        unless aws_s3.bucket_exists?(bucket_name)
+          aws_s3.create_bucket(bucket_name)
+        end
+      end
+
+      def s3_safe_vpc_domain
+        @config['vpc']['domain'].gsub('.', '-')
+      end
+
+      def bucket_name
+        if aws_s3.bucket_exists?("#{@config['name']}-bosh-artifacts")
+          "#{@config['name']}-bosh-artifacts"
+        else
+          "#{s3_safe_vpc_domain}-bosh-artifacts"
+        end
+      end
+
+      def migrations_name
+        "aws_migrations/migrations.yaml"
+      end
+
+      def run_migrations(migrations_to_run)
+        migrations_to_run.each do |migration|
+          migration.load_class.new(@config, bucket_name).run
+          record_migration(migration)
+        end
+      end
+
+      def load_migrations
+        Dir.glob(File.join(migration_path, "*.rb")).collect do |migration_file_path|
+          version, name  = migration_file_path.scan(/([0-9]+)_([_a-z0-9]*).rb\z/).first
+          MigrationProxy.new(name,version.to_i)
+        end.sort
+      end
+
+      def load_migrations_for_env
+        yaml_file = aws_s3.fetch_object_contents(bucket_name, migrations_name) || ""
+        migrations = YAML.load(yaml_file) || []
+
+        migrations.collect do |migration_yaml|
+          MigrationProxy.new(migration_yaml['name'],migration_yaml['version'].to_i)
+        end.sort
+      end
+
+      def record_migration(migration)
+        environment_migrations << migration
+        migration_yaml = YAML.dump(environment_migrations.collect do |m|
+          m.to_hash
+        end)
+        aws_s3.upload_to_bucket(bucket_name, migrations_name, migration_yaml)
+      end
+    end
+
+    class MigrationProxy
+      include Comparable
+
+      attr_reader :name, :version
+
+      def initialize(name, version)
+        @name = name
+        @version = version.to_i
+      end
+
+      def load_class
+        require File.join(MigrationHelper.aws_migration_directory, "#{version}_#{name}")
+        Object.const_get(MigrationHelper.to_class_name(name))
+      end
+
+      def eql?(other)
+        version == other.version
+      end
+
+      def <=>(other)
+        version <=> other.version
+      end
+
+      def hash
+        (name.to_s + version.to_s).hash
+      end
+
+      def to_hash
+        {"name" => name, "version" => version}
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/rds.rb

@@ -0,0 +1,182 @@
+require "securerandom"
+
+module Bosh
+  module Aws
+    class RDS
+      DEFAULT_RDS_OPTIONS = {
+          :allocated_storage => 5,
+          :db_instance_class => "db.t1.micro",
+          :engine => "mysql",
+          :multi_az => true,
+          :engine_version => "5.5.31"
+      }
+      DEFAULT_RDS_PROTOCOL = :tcp
+      DEFAULT_MYSQL_PORT = 3306
+
+      def initialize(credentials)
+        @credentials = credentials
+        @aws_provider = AwsProvider.new(@credentials)
+      end
+
+      def create_database(name, subnet_ids, vpc_id, options = {})
+        create_db_parameter_group('utf8')
+        vpc = Bosh::Aws::VPC.find(Bosh::Aws::EC2.new(@credentials), vpc_id)
+        create_vpc_db_security_group(vpc, name) if vpc.security_group_by_name(name).nil?
+        create_subnet_group(name, subnet_ids) unless subnet_group_exists?(name)
+
+        # symbolize options keys
+        options = options.inject({}) { |memo, (k, v)| memo[k.to_sym] = v; memo }
+
+        creation_options = DEFAULT_RDS_OPTIONS.merge(options)
+        creation_options[:db_instance_identifier] = name
+        creation_options[:db_name]              ||= name
+        creation_options[:vpc_security_group_ids] = [vpc.security_group_by_name(name).id]
+        creation_options[:db_subnet_group_name]   = name
+        creation_options[:db_parameter_group_name]     = 'utf8'
+        creation_options[:master_username]      ||= generate_user
+        creation_options[:master_user_password] ||= generate_password
+        response = aws_rds_client.create_db_instance(creation_options)
+        response.data.merge(:master_user_password => creation_options[:master_user_password])
+      end
+
+      def databases
+        aws_rds.db_instances
+      end
+
+      def database(name)
+        databases.find { |v| v.id == name }
+      end
+
+      def database_exists?(name)
+        !database(name).nil?
+      end
+
+      def delete_databases
+        databases.each { |db| db.delete(skip_final_snapshot: true) unless db.db_instance_status == "deleting" }
+      end
+
+      def database_names
+        databases.inject({}) do |memo, db_instance|
+          memo[db_instance.id] = db_instance.name
+          memo
+        end
+      end
+
+      def subnet_group_exists?(name)
+        aws_rds_client.describe_db_subnet_groups(:db_subnet_group_name => name)
+        return true
+      rescue AWS::RDS::Errors::DBSubnetGroupNotFoundFault
+        return false
+      end
+
+      def db_parameter_group_names
+        charset = 'utf8'
+        param_names = %w(character_set_server
+                         character_set_client
+                         character_set_results
+                         character_set_database
+                         character_set_connection)
+
+        params = param_names.map{|param_name| {:parameter_name => param_name,
+                                      :parameter_value => charset,
+                                      :apply_method => 'immediate'}}
+
+        params << {:parameter_name => 'collation_connection',
+                                               :parameter_value => 'utf8_unicode_ci',
+                                               :apply_method => 'immediate'}
+        params << {:parameter_name => 'collation_server',
+                                           :parameter_value => 'utf8_unicode_ci',
+                                           :apply_method => 'immediate'}
+        params
+      end
+
+      def create_db_parameter_group(name)
+        aws_rds_client.describe_db_parameter_groups(:db_parameter_group_name => name)
+      rescue AWS::RDS::Errors::DBParameterGroupNotFound
+        aws_rds_client.create_db_parameter_group(:db_parameter_group_name => name,
+        :db_parameter_group_family => 'mysql5.5', :description => name)
+        aws_rds_client.modify_db_parameter_group(:db_parameter_group_name => name,
+            :parameters => db_parameter_group_names)
+      end
+
+      def delete_db_parameter_group(name)
+        aws_rds_client.describe_db_parameter_groups(:db_parameter_group_name => name)
+        aws_rds_client.delete_db_parameter_group(:db_parameter_group_name => name)
+      rescue AWS::RDS::Errors::DBParameterGroupNotFound
+      end
+
+      def create_subnet_group(name, subnet_ids)
+        aws_rds_client.create_db_subnet_group(
+            :db_subnet_group_name => name,
+            :db_subnet_group_description => name,
+            :subnet_ids => subnet_ids
+        )
+      end
+
+      def subnet_group_names
+        aws_rds_client.describe_db_subnet_groups.data[:db_subnet_groups].map { |sg| sg[:db_subnet_group_name] }
+      end
+
+      def delete_subnet_group(name)
+        aws_rds_client.delete_db_subnet_group(:db_subnet_group_name => name)
+      end
+
+      def delete_subnet_groups
+        subnet_group_names.each { |name| delete_subnet_group(name) }
+      end
+
+      def create_vpc_db_security_group(vpc, name)
+        group_spec = {
+            "name" => name,
+            "ingress" => [
+                {
+                    "ports" => DEFAULT_MYSQL_PORT,
+                    "protocol" => DEFAULT_RDS_PROTOCOL,
+                    "sources" => vpc.cidr_block,
+                },
+            ],
+        }
+
+        vpc.create_security_groups([group_spec])
+      end
+
+      def security_group_names
+        aws_rds_client.describe_db_security_groups.data[:db_security_groups].map { |sg| sg[:db_security_group_name] }
+      end
+
+      def delete_security_group(name)
+        aws_rds_client.delete_db_security_group(:db_security_group_name => name)
+      end
+
+      def delete_security_groups
+        security_group_names.each do |name|
+          delete_security_group(name) unless name == "default"
+        end
+      end
+
+      def aws_rds
+        aws_provider.rds
+      end
+
+      def aws_rds_client
+        aws_provider.rds_client
+      end
+
+      private
+
+      attr_reader :aws_provider
+
+      def generate_user
+        generate_credential("u", 7)
+      end
+
+      def generate_password
+        generate_credential("p", 16)
+      end
+
+      def generate_credential(prefix, length)
+        "#{prefix}#{SecureRandom.hex(length)}"
+      end
+    end
+  end
+end