bosh_cli_plugin_aws 1.5.0.pre.1113

data/lib/bosh_cli_plugin_aws/route53.rb

@@ -0,0 +1,103 @@
+module Bosh
+  module Aws
+    class Route53
+
+      def initialize(credentials)
+        @aws_provider = AwsProvider.new(credentials)
+      end
+
+      def create_zone(zone)
+        zone = "#{zone}." unless zone =~ /\.$/
+        aws_route53.client.create_hosted_zone(:name => zone, :caller_reference => generate_unique_name)
+        true
+      end
+
+      def delete_zone(zone)
+        zone = "#{zone}." unless zone =~ /\.$/
+        aws_route53.client.delete_hosted_zone(:id => get_zone_id(zone))
+        true
+      end
+
+      def add_record(host, zone, addresses, options={})
+        host = "\\052" if host == "*"
+        zone = "#{zone}." unless zone =~ /\.$/
+        addresses = [addresses] unless addresses.kind_of?(Array)
+        type = options[:type] || "A"
+        ttl = options[:ttl] || 3600
+        aws_route53.client.change_resource_record_sets(
+          hosted_zone_id: get_zone_id(zone),
+          change_batch: {
+            changes: [
+              {
+                action: "CREATE",
+                resource_record_set: {
+                  name: "#{host}.#{zone}",
+                  type: type,
+                  ttl: ttl,
+                  resource_records: addresses.map {|address| { value: address} }
+                }
+              }
+            ]
+          }
+        )
+        true
+      end
+
+      def delete_record(host, zone, options={})
+        host = "\\052" if host == "*"
+        zone = "#{zone}." unless zone =~ /\.$/
+        record_name = "#{host}.#{zone}"
+        record_type = options[:type] || "A"
+
+        zone_response = aws_route53.client.list_resource_record_sets(:hosted_zone_id => get_zone_id(zone))
+        resource_record_set = zone_response.data[:resource_record_sets].find do |rr|
+          rr[:name] == record_name && rr[:type] == record_type
+        end
+
+        unless resource_record_set
+          raise "no #{record_type} record found for #{record_name}"
+        end
+        aws_route53.client.change_resource_record_sets(
+          hosted_zone_id: get_zone_id(zone),
+          change_batch: {
+            changes: [
+              {
+                action: "DELETE",
+                resource_record_set: resource_record_set
+              }
+            ]
+          }
+        )
+        true
+      end
+
+      def delete_all_records(options = {})
+        omit_types = options[:omit_types] || []
+        aws_route53.hosted_zones.each do |zone|
+          zone.rrsets.each do |rs|
+            rs.delete unless omit_types.include?(rs.type)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :aws_provider
+
+      def aws_route53
+        aws_provider.route53
+      end
+
+      def get_zone_id(name)
+        zones_response = aws_route53.client.list_hosted_zones
+        zone = zones_response.data[:hosted_zones].find { |zone| zone[:name] == name }
+        raise "Zone not found for #{name} in route53 zones response #{zones_response.inspect}" if zone.nil?
+        zone.fetch(:id)
+      end
+
+      def generate_unique_name
+        SecureRandom.uuid
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/s3.rb

@@ -0,0 +1,93 @@
+module Bosh
+  module Aws
+    class S3
+      def initialize(credentials)
+        @aws_provider = AwsProvider.new(credentials)
+      end
+
+      def create_bucket(bucket_name)
+        aws_s3.buckets.create(bucket_name)
+      end
+
+      def move_bucket(old_bucket, new_bucket)
+        fetch_bucket(old_bucket).objects.each do |object|
+          object.move_to(object.key, :bucket_name => new_bucket)
+        end
+      end
+
+      def delete_bucket(bucket_name)
+        bucket = fetch_bucket(bucket_name)
+
+        bucket.clear!
+        bucket.delete
+      rescue AWS::S3::Errors::NoSuchBucket
+      end
+
+      def empty
+        aws_s3.buckets.each do |bucket|
+          begin
+            bucket.delete!
+          rescue AWS::S3::Errors::NoSuchBucket
+            # when the bucket goes away while going through the list
+          end
+        end
+      end
+
+      def bucket_names
+        aws_s3.buckets.map &:name
+      end
+
+      def bucket_exists?(bucket_name)
+        bucket_names.include?(bucket_name)
+      end
+
+      def upload_to_bucket(bucket_name, object_name, io)
+        bucket = fetch_bucket(bucket_name)
+        bucket.objects[object_name].write(io)
+      end
+
+      def objects_in_bucket(bucket_name)
+        fetch_bucket(bucket_name).objects.map { |object| object.key }
+      end
+
+      def fetch_object_contents(bucket_name, object_name)
+        bucket = fetch_bucket(bucket_name)
+        Bosh::Common.retryable(on: AWS::S3::Errors::NoSuchBucket, tries: 10) do
+          bucket.objects[object_name].read
+        end
+      rescue AWS::S3::Errors::NoSuchKey
+        nil
+      end
+
+      def copy_remote_file(bucket_name, remote_file, file_name)
+        say("Fetching remote file #{remote_file} from #{bucket_name} bucket")
+        bucket = aws_s3.buckets[bucket_name]
+        object = bucket.objects[remote_file]
+        release_file = Tempfile.new file_name
+        Bosh::Cli::FileWithProgressBar.open(release_file, 'wb') do |f|
+          f.size=object.content_length
+          object.read do |chunk|
+            f.write chunk
+          end
+        end
+        release_file
+      rescue AWS::S3::Errors::NoSuchKey => e
+        new_exception = Exception.new("Can't find #{remote_file} in bucket #{bucket_name}")
+        new_exception.set_backtrace(e.backtrace)
+        raise new_exception
+      end
+
+      private
+
+      attr_reader :aws_provider
+
+      def fetch_bucket(bucket_name)
+        aws_s3.buckets[bucket_name]
+      end
+
+      def aws_s3
+        aws_provider.s3
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws/version.rb

@@ -0,0 +1,5 @@
+module Bosh
+  module Aws
+    VERSION = '1.5.0.pre.1113'
+  end
+end

data/lib/bosh_cli_plugin_aws/vpc.rb

@@ -0,0 +1,181 @@
+module Bosh
+  module Aws
+    class VPC
+
+      DEFAULT_CIDR = "10.0.0.0/16"
+      DEFAULT_ROUTE = "0.0.0.0/0"
+      NAT_INSTANCE_DEFAULTS = {
+          :image_id => "ami-f619c29f",
+          :instance_type => "m1.small"
+      }
+
+      def initialize(ec2, aws_vpc)
+        @ec2 = ec2
+        @aws_vpc = aws_vpc
+      end
+
+      def self.create(ec2, cidr = DEFAULT_CIDR, instance_tenancy = nil)
+        vpc_options = instance_tenancy ? {instance_tenancy: instance_tenancy} : {}
+        self.new(ec2, ec2.vpcs.create(cidr, vpc_options))
+      end
+
+      def self.find(ec2, vpc_id)
+        self.new(ec2, ec2.vpcs[vpc_id])
+      end
+
+      def make_internet_gateway_default_route_for_subnet(subnet)
+        route_table = @aws_vpc.route_tables.create
+        route_table.create_route(DEFAULT_ROUTE, internet_gateway: @aws_vpc.internet_gateway)
+        subnet.route_table = route_table
+      end
+
+      def make_nat_instance_default_route_for_subnet(subnet, nat_instance)
+        route_table = @aws_vpc.route_tables.create
+        route_table.create_route(DEFAULT_ROUTE, instance: nat_instance)
+        subnet.route_table = route_table
+      end
+
+      def vpc_id
+        @aws_vpc.id
+      end
+
+      def cidr_block
+        @aws_vpc.cidr_block
+      end
+
+      def instances_count
+        @aws_vpc.instances.count
+      end
+
+      def dhcp_options
+        @aws_vpc.dhcp_options
+      end
+
+      def state
+        @aws_vpc.state
+      end
+
+      def subnets
+        Hash[@aws_vpc.subnets.map { |subnet| [subnet.tags["Name"], subnet.id] }]
+      end
+
+      def delete_vpc
+        @aws_vpc.delete
+        Bosh::Common.retryable(tries: 30, sleep: 5, on: []) do
+          begin
+            false if @aws_vpc.state
+          rescue AWS::EC2::Errors::InvalidVpcID::NotFound
+            true
+          end
+        end
+
+      rescue ::AWS::EC2::Errors::DependencyViolation
+        err "#{@aws_vpc.id} has dependencies that this tool does not delete"
+      end
+
+      def create_security_groups(groups_specs)
+        groups_specs.each do |group_spec|
+          if group_name_available group_spec["name"]
+            security_group = @aws_vpc.security_groups.create(group_spec["name"])
+            Bosh::AwsCloud::ResourceWait.for_sgroup(sgroup: security_group, state: true)
+            group_spec["ingress"].each do |ingress|
+              range_match = ingress["ports"].to_s.match(/(\d+)\s*-\s*(\d+)/)
+              ports = range_match ? (range_match[1].to_i)..(range_match[2].to_i) : ingress["ports"].to_i
+              security_group.authorize_ingress(ingress["protocol"], ports, ingress["sources"])
+            end
+          end
+        end
+      end
+
+      def delete_security_groups
+        @aws_vpc.security_groups.reject { |group| group.name == "default" }.each(&:delete)
+      end
+
+      def security_group_by_name(name)
+        @aws_vpc.security_groups.detect { |sg| sg.name == name }
+      end
+
+      def create_subnets(subnets)
+        subnets.each_pair do |name, subnet_spec|
+          yield "Making subnet #{name} #{subnet_spec["cidr"]}:" if block_given?
+          options = {}
+          options[:availability_zone] = subnet_spec["availability_zone"] if subnet_spec["availability_zone"]
+
+          subnet = @aws_vpc.subnets.create(subnet_spec["cidr"], options)
+          Bosh::AwsCloud::ResourceWait.for_subnet(subnet: subnet, state: :available)
+
+          subnet.add_tag("Name", :value => name)
+        end
+      end
+
+      def extract_nat_instance_specs(specs)
+        subnet_specs_with_nats = specs.select do |_, subnet_spec|
+          subnet_spec.has_key?("nat_instance")
+        end
+
+        subnet_specs_with_nats.map do |subnet_name, subnet_spec|
+          nat_instance_spec = subnet_spec["nat_instance"]
+          nat_instance_spec["subnet_id"] = subnets[subnet_name]
+          nat_instance_spec
+        end
+      end
+
+      def create_nat_instances(subnets)
+        extract_nat_instance_specs(subnets).each do |subnet_spec|
+          @ec2.create_nat_instance(subnet_spec)
+        end
+      end
+
+      def setup_subnet_routes(subnet_specs)
+        subnet_specs.each_pair do |name, subnet_spec|
+          if subnet_spec["default_route"]
+            subnet = @aws_vpc.subnets[subnets[name]]
+            yield "  Making routing table for #{name}" if block_given?
+            yield "  Binding default route to #{subnet_spec["default_route"]}" if block_given?
+            if subnet_spec["default_route"] == "igw"
+              make_internet_gateway_default_route_for_subnet(subnet)
+            else
+              make_nat_instance_default_route_for_subnet(subnet, @ec2.get_running_instance_by_name(subnet_spec["default_route"]))
+            end
+          end
+        end
+      end
+
+      def delete_subnets
+        @aws_vpc.subnets.each(&:delete)
+      end
+
+      def delete_route_tables
+        @aws_vpc.route_tables.reject(&:main?).each(&:delete)
+      end
+
+      def delete_network_interfaces
+        @aws_vpc.network_interfaces.each(&:delete)
+      end
+
+      def create_dhcp_options(options)
+        default_dhcp_opts = @aws_vpc.dhcp_options
+
+        new_dhcp_options = @ec2.dhcp_options.create(options)
+        new_dhcp_options.associate(vpc_id)
+        #say "\tcreated and associated DHCP options #{new_dhcp_options.id}".make_green
+
+        default_dhcp_opts.delete
+      end
+
+      def attach_internet_gateway(gateway_id)
+        @aws_vpc.internet_gateway = gateway_id
+      end
+
+      private
+
+      def group_name_available(name)
+        @aws_vpc.security_groups.each { |group| group.delete if group.name == name }
+        true
+      rescue ::AWS::EC2::Errors::DependencyViolation => e
+        say "unable to delete security group: #{name}: #{e.message}".make_yellow
+        false
+      end
+    end
+  end
+end

data/lib/bosh_cli_plugin_aws.rb

@@ -0,0 +1,31 @@
+require 'aws-sdk'
+require 'logger'
+
+# since this plugin is abusing the AWS CPI, the following hack is needed so that
+# task_checkpoint & logger is available when ResourceWait.for_resource is called
+require "cloud"
+Config = Struct.new(:task_checkpoint, :logger)
+Bosh::Clouds::Config.configure(Config.new(true, Logger.new(File::NULL)))
+
+require "cloud/aws/resource_wait"
+require "common/ssl"
+
+require "bosh_cli_plugin_aws/version"
+require "bosh_cli_plugin_aws/ec2"
+require "bosh_cli_plugin_aws/route53"
+require "bosh_cli_plugin_aws/s3"
+require "bosh_cli_plugin_aws/vpc"
+require "bosh_cli_plugin_aws/rds"
+require "bosh_cli_plugin_aws/elb"
+require "bosh_cli_plugin_aws/bosh_bootstrap"
+require "bosh_cli_plugin_aws/micro_bosh_bootstrap"
+require "bosh_cli_plugin_aws/aws_config"
+require 'bosh_cli_plugin_aws/aws_provider'
+require "bosh/cli/commands/aws"
+require "bosh/cli/commands/micro"
+require "bosh_cli_plugin_aws/microbosh_manifest"
+require "bosh_cli_plugin_aws/bat_manifest"
+require "bosh_cli_plugin_aws/bosh_manifest"
+require "bosh_cli_plugin_aws/migration_helper"
+require "bosh_cli_plugin_aws/migration"
+require "bosh_cli_plugin_aws/migrator"

data/migrations/20130412000811_create_key_pairs.rb

@@ -0,0 +1,8 @@
+class CreateKeyPairs < Bosh::Aws::Migration
+  def execute
+    say "allocating #{config["key_pairs"].length} KeyPair(s)"
+    config["key_pairs"].each do |name, path|
+      ec2.force_add_key_pair(name, path)
+    end
+  end
+end

data/migrations/20130412004642_create_vpc.rb

@@ -0,0 +1,65 @@
+class CreateVpc < Bosh::Aws::Migration
+  def execute
+    receipt = {}
+
+    receipt["aws"] = config["aws"]
+
+    vpc = Bosh::Aws::VPC.create(ec2, config["vpc"]["cidr"], config["vpc"]["instance_tenancy"])
+    receipt["vpc"] = {"id" => vpc.vpc_id, "domain" => config["vpc"]["domain"]}
+
+    receipt["original_configuration"] = config
+
+    unless was_vpc_eventually_available?(vpc)
+      err "VPC #{vpc.vpc_id} was not available within 60 seconds, giving up"
+    end
+
+    say "creating internet gateway"
+    igw = ec2.create_internet_gateway
+    vpc.attach_internet_gateway(igw.id)
+
+    security_groups = config["vpc"]["security_groups"]
+    say "creating security groups: #{security_groups.map { |group| group["name"] }.join(", ")}"
+    vpc.create_security_groups(security_groups)
+
+    subnets = config["vpc"]["subnets"]
+    say "creating subnets: #{subnets.keys.join(", ")}"
+    vpc.create_subnets(subnets) { |msg| say "  #{msg}" }
+    vpc.create_nat_instances(subnets)
+    vpc.setup_subnet_routes(subnets) { |msg| say "  #{msg}" }
+    receipt["vpc"]["subnets"] = vpc.subnets
+
+    elbs = config["vpc"]["elbs"]
+    ssl_certs = config["ssl_certs"]
+
+    say "creating load balancers: #{elbs.keys.join(", ")}" if elbs
+    elbs.each do |name, settings|
+      settings["domain"] = config["vpc"]["domain"]
+      e = elb.create(name, vpc, settings, ssl_certs)
+      if settings["dns_record"]
+        say "adding CNAME record for #{settings["dns_record"]}.#{config["vpc"]["domain"]}"
+        route53.add_record(settings["dns_record"], config["vpc"]["domain"], [e.dns_name], {ttl: settings["ttl"], type: 'CNAME'})
+      end
+    end
+
+    dhcp_options = config["vpc"]["dhcp_options"]
+    say "creating DHCP options"
+    vpc.create_dhcp_options(dhcp_options)
+  rescue Bosh::Aws::ELB::BadCertificateError => e
+    err e.message
+  ensure
+    save_receipt("aws_vpc_receipt", receipt)
+  end
+
+  private
+
+  def was_vpc_eventually_available?(vpc)
+    (1..60).any? do |attempt|
+      begin
+        sleep 1 unless attempt == 1
+        vpc.state.to_s == "available"
+      rescue Exception => e
+        say("Waiting for vpc, continuing after #{e.class}: #{e.message}")
+      end
+    end
+  end
+end

data/migrations/20130412181302_create_route53_records.rb

@@ -0,0 +1,37 @@
+class CreateRoute53Records < Bosh::Aws::Migration
+  def execute
+    receipt = {}
+    elastic_ip_specs = config["elastic_ips"]
+
+    if elastic_ip_specs
+      receipt["elastic_ips"] = {}
+    else
+      return
+    end
+
+    count = elastic_ip_specs.map{|_, spec| spec["instances"]}.inject(:+)
+    say "allocating #{count} elastic IP(s)"
+    ec2.allocate_elastic_ips(count)
+
+    elastic_ips = ec2.elastic_ips
+
+    elastic_ip_specs.each do |name, job|
+      receipt["elastic_ips"][name] = {"ips" => elastic_ips.shift(job["instances"])}
+    end
+
+    elastic_ip_specs.each do |name, job|
+      if job["dns_record"]
+        say "adding A record for #{job["dns_record"]}.#{config["vpc"]["domain"]}"
+        route53.add_record(
+            job["dns_record"],
+            config["vpc"]["domain"],
+            receipt["elastic_ips"][name]["ips"],
+            {ttl: job["ttl"]}
+        ) # shouldn't have to get domain from config["vpc"]["domain"]; should use config["name"]
+        receipt["elastic_ips"][name]["dns_record"] = job["dns_record"]
+      end
+    end
+  ensure
+    save_receipt("aws_route53_receipt", receipt)
+  end
+end

data/migrations/20130412183544_create_rds_dbs.rb

@@ -0,0 +1,35 @@
+class CreateRdsDbs < Bosh::Aws::Migration
+  include Bosh::Aws::MigrationHelper
+
+  def execute
+    if !config["rds"]
+      say "rds not set in config.  Skipping"
+      return
+    end
+
+    vpc_receipt = load_receipt("aws_vpc_receipt")
+    db_names   = %w(ccdb uaadb)
+    db_configs = config['rds'].select {|c| db_names.include?(c['instance']) }
+    RdsDb.aws_rds = rds
+    dbs = []
+
+    begin
+      db_configs.each do |rds_db_conf|
+        rds_args = { vpc_receipt: vpc_receipt, rds_db_conf: rds_db_conf }
+        rds_db = RdsDb.new(rds_args)
+        dbs << rds_db
+        rds_db.create!
+      end
+
+      if RdsDb.was_rds_eventually_available?
+        dbs.each { |db| db.update_receipt }
+      else
+        err "RDS was not available within 60 minutes, giving up"
+      end
+
+    ensure
+      save_receipt("aws_rds_receipt", RdsDb.receipt)
+    end
+  end
+
+end

data/migrations/20130412192351_create_s3.rb

@@ -0,0 +1,4 @@
+class CreateS3 < Bosh::Aws::Migration
+  def execute
+  end
+end

data/migrations/20130529212130_create_more_unique_s3_buckets.rb

@@ -0,0 +1,33 @@
+class CreateMoreUniqueS3Buckets < Bosh::Aws::Migration
+  def s3_safe_full_domain_name
+    config['vpc']['domain'].gsub(".","-")
+  end
+
+  def old_prefix
+    config['name']
+  end
+
+  def buckets
+    {
+     "#{s3_safe_full_domain_name}-bosh-blobstore" => "#{old_prefix}-bosh-blobstore",
+     "#{s3_safe_full_domain_name}-bosh-artifacts" => "#{old_prefix}-bosh-artifacts"
+    }
+  end
+
+  def execute
+    return if s3_safe_full_domain_name == old_prefix
+
+    buckets.each_key do |bucket|
+      say "creating bucket #{bucket}"
+      s3.create_bucket(bucket)
+    end
+
+    buckets.each_pair do |new_bucket, old_bucket|
+      next unless s3.bucket_exists?(old_bucket)
+      say "moving contents of #{old_bucket} to #{new_bucket}"
+      s3.move_bucket(old_bucket, new_bucket)
+      say "deleting bucket #{old_bucket}"
+      s3.delete_bucket(old_bucket)
+    end
+  end
+end

data/migrations/20130531180445_create_bosh_rds_db.rb

@@ -0,0 +1,30 @@
+class CreateBoshRdsDb < Bosh::Aws::Migration
+  include Bosh::Aws::MigrationHelper
+
+  def execute
+    vpc_receipt = load_receipt("aws_vpc_receipt")
+    db_names   = %w(bosh)
+    db_configs = config['rds'].select {|c| db_names.include?(c['instance']) }
+    RdsDb.aws_rds = rds
+    dbs = []
+
+    begin
+      db_configs.each do |rds_db_conf|
+        rds_args = { vpc_receipt: vpc_receipt, rds_db_conf: rds_db_conf }
+        rds_db = RdsDb.new(rds_args)
+        dbs << rds_db
+        rds_db.create!
+      end
+
+      if RdsDb.was_rds_eventually_available?
+        dbs.each { |db| db.update_receipt }
+      else
+        err "RDS was not available within 60 minutes, giving up"
+      end
+
+    ensure
+      save_receipt("aws_rds_bosh_receipt", RdsDb.receipt)
+    end
+  end
+
+end