bosh_cli_plugin_aws 1.5.0.pre.1113

data/migrations/20130826150635_update_elb_for_websockets.rb

@@ -0,0 +1,97 @@
+class UpdateElbForWebsockets < Bosh::Aws::Migration
+  def execute
+    validate_receipt
+
+    vpc = Bosh::Aws::VPC.find(ec2, vpc_id)
+    security_group = vpc.security_group_by_name(cfrouter_security_group_name)
+
+    params = {"protocol" => "tcp", "ports" => "4443", "sources" => "0.0.0.0/0"}
+    if WebSocketElbHelpers.authorize_ingress(security_group, params)
+      WebSocketElbHelpers.record_ingress(vpc_receipt, cfrouter_security_group_name, params)
+      save_receipt('aws_vpc_receipt', vpc_receipt)
+    end
+
+    cfrouter_elb = elb.find_by_name("cfrouter")
+
+    params = {port: 443, protocol: :https}
+    https_listener_server_certificate = WebSocketElbHelpers.find_server_certificate_from_listeners(cfrouter_elb, params)
+
+    params = {port: 4443, protocol: :ssl, instance_port: 80, instance_protocol: :tcp, server_certificate: https_listener_server_certificate}
+    WebSocketElbHelpers.create_listener(cfrouter_elb, params)
+  end
+
+  private
+
+  def validate_receipt
+    begin
+      cfrouter_config
+    rescue KeyError
+      err("Unable to find `cfrouter' ELB configuration in AWS VPC Receipt")
+    end
+
+    begin
+      cfrouter_security_group_name
+    rescue KeyError
+      err("Unable to find `cfrouter' ELB Security Group in AWS VPC Receipt")
+    end
+
+    begin
+      vpc_id
+    rescue KeyError
+      err("Unable to find VPC ID in AWS VPC Receipt")
+    end
+
+  end
+
+  def vpc_receipt
+    @vpc_receipt ||= load_receipt('aws_vpc_receipt')
+  end
+
+  def cfrouter_config
+    vpc_receipt.fetch('original_configuration').fetch('vpc').fetch('elbs').fetch('cfrouter')
+  end
+
+  def cfrouter_security_group_name
+    cfrouter_config.fetch('security_group')
+  end
+
+  def vpc_id
+    vpc_receipt.fetch('vpc').fetch('id')
+  end
+
+  class WebSocketElbHelpers
+    def self.find_security_group_by_name(ec2, vpc_id, name)
+      vpc = Bosh::Aws::VPC.find(ec2, vpc_id)
+      security_group = vpc.security_group_by_name(name)
+
+      err("AWS reports that security group #{name} does not exist") unless security_group
+      security_group
+    end
+
+    def self.authorize_ingress(security_group, params)
+      security_group.authorize_ingress(params['protocol'], params['ports'].to_i, params['sources'])
+      true
+    rescue AWS::EC2::Errors::InvalidPermission::Duplicate
+      false
+    end
+
+    def self.record_ingress(vpc_receipt, security_group_name, params)
+        receipt_security_groups = vpc_receipt['original_configuration']['vpc']['security_groups']
+        receipt_router_security_group = receipt_security_groups.find{ |g| g['name'] == security_group_name}
+        receipt_router_security_group['ingress'] << params
+    end
+
+    def self.find_server_certificate_from_listeners(elb, params)
+      listener = elb.listeners.find {|l| l.port == params[:port] && l.protocol == params[:protocol] }
+
+      err("Could not find listener with params `#{params.inspect}' on ELB `#{elb.name}'") unless listener
+      err("Could not find server certificate for listener with params `#{params.inspect}' on ELB `#{elb.name}'") unless listener.server_certificate
+
+      listener.server_certificate
+    end
+
+    def self.create_listener(elb, params)
+      elb.listeners.create(params)
+    end
+  end
+end

data/migrations/20130827000001_add_secondary_az_to_vpc.rb

@@ -0,0 +1,34 @@
+class AddSecondaryAzToVpc < Bosh::Aws::Migration
+  include Bosh::Aws::MigrationHelper
+
+  def execute
+    vpc_receipt = load_receipt("aws_vpc_receipt")
+
+    vpc = Bosh::Aws::VPC.find(ec2, vpc_receipt["vpc"]["id"])
+
+    new_az = vpc_receipt["original_configuration"]["vpc"]["subnets"]["cf_elb2"]["availability_zone"]
+
+    subnets = {
+      "bosh2" => {"availability_zone" => new_az, "cidr" => "10.10.64.0/24", "default_route" => "igw"},
+      "cf2" => {"availability_zone" => new_az, "cidr" => "10.10.80.0/20", "default_route" => "cf_nat_box1"},
+      "services2" => {"availability_zone" => new_az, "cidr" => "10.10.96.0/20", "default_route" => "cf_nat_box1"},
+    }
+
+    existing_subnets = vpc.subnets
+
+    subnets.reject! { |subnet, _|
+      existing_subnets.include?(subnet).tap do |should_skip|
+        say "  Skipping already-present subnet #{subnet.inspect}" if should_skip
+      end
+    }
+
+
+    vpc.create_subnets(subnets) { |msg| say "  #{msg}" }
+    vpc.create_nat_instances(subnets)
+    vpc.setup_subnet_routes(subnets) { |msg| say "  #{msg}" }
+
+    vpc_receipt["vpc"]["subnets"] = vpc.subnets
+  ensure
+    save_receipt("aws_vpc_receipt", vpc_receipt)
+  end
+end

data/templates/aws_configuration_template.yml.erb

@@ -0,0 +1,187 @@
+---
+aws:
+  secret_access_key: <%= aws_secret_access_key %>
+  access_key_id: <%= aws_access_key_id %>
+  region: <%= aws_region %>
+name: <%= vpc_deployment_name %>
+vpc:
+  domain: <%= vpc_generated_domain %>
+  instance_tenancy: default
+  cidr: 10.10.0.0/16
+  subnets:
+    <%- [[vpc_primary_az, 0, 1], [vpc_secondary_az, 64, 2]].each do |(az, third_octet, index)| -%>
+    <%- if index == 1 -%> # only the first subnet gets a NAT box or CF/Services nodes until we want to start using them
+    bosh<%= index %>:
+      cidr: 10.10.<%= third_octet + 0 %>.0/24
+      availability_zone: <%= az %>
+      default_route: igw
+      nat_instance:
+        name: cf_nat_box<%= index %>
+        ip: 10.10.0.10  # spin up NAT instance at fixed IP of 10.10.0.10
+        security_group: open
+        key_name: <%= key_pair_name %>
+        <%- if production_resources? -%>
+        instance_type: m1.xlarge
+        <%- end -%>
+    <%- end -%>
+    bosh_rds<%= index %>:
+      cidr: 10.10.<%= third_octet + 1 %>.0/24
+      availability_zone: <%= az %>
+    cf_elb<%= index %>:
+      cidr: 10.10.<%= third_octet + 2 %>.0/24
+      availability_zone: <%= az %>
+      default_route: igw
+    cf_rds<%= index %>:
+      cidr: 10.10.<%= third_octet + 3 %>.0/24
+      availability_zone: <%= az %>
+    services_rds<%= index %>:
+      cidr: 10.10.<%= third_octet + 8 %>.0/21
+      availability_zone: <%= az %>
+    <%- if index == 1 -%>
+    cf<%= index %>:
+      cidr: 10.10.<%= third_octet + 16 %>.0/20
+      availability_zone: <%= az %>
+      default_route: cf_nat_box<%= index %>
+    services<%= index %>:
+      cidr: 10.10.<%= third_octet + 32 %>.0/20
+      availability_zone: <%= az %>
+      default_route: cf_nat_box<%= index %>
+    <%- end -%>
+    <%- end -%>
+  dhcp_options:
+    domain_name_servers:
+      - 10.10.0.6  # IP of the BOSH DNS server?
+      - 10.10.0.2  # local amazon public DNS server
+  security_groups:
+    - name: open
+      ingress:
+        - protocol: tcp
+          ports: '0 - 65535'
+          sources: 0.0.0.0/0
+        - protocol: udp
+          ports: '0 - 65535'
+          sources: 0.0.0.0/0
+    - name: bosh
+      ingress:
+        - protocol: tcp
+          ports: '0 - 65535'
+          sources: 0.0.0.0/0
+        - protocol: udp
+          ports: '0 - 65535'
+          sources: 0.0.0.0/0
+    - name: bat
+      ingress:
+        - protocol: tcp
+          ports: '4567'
+          sources: 0.0.0.0/0
+        - protocol: tcp
+          ports: '22'
+          sources: 0.0.0.0/0
+    - name: cf
+      ingress:
+        - protocol: tcp
+          ports: '0 - 65535'
+          sources: 0.0.0.0/0
+        - protocol: udp
+          ports: '0 - 65535'
+          sources: 0.0.0.0/0
+    - name: web
+      ingress:
+        - protocol: tcp
+          ports: '80'
+          sources: 0.0.0.0/0
+        - protocol: tcp
+          ports: '443'
+          sources: 0.0.0.0/0
+  elbs:
+    cfrouter:
+      dns_record: "*"
+      ttl: 60
+      subnets:
+      - cf_elb1
+      - cf_elb2
+      security_group: web
+      https: true
+      ssl_cert: cfrouter_cert
+
+ssl_certs:
+  director_cert:
+    private_key_path: <%= director_ssl_key_file %>
+    certificate_path: <%= director_ssl_cert_file %>
+  cfrouter_cert:
+    private_key_path: <%= elb_ssl_key_file %>
+    certificate_path: <%= elb_ssl_cert_file %>
+    certificate_chain_path: <%= elb_ssl_cert_chain_file %>
+
+elastic_ips:
+  # each NAT box automatically reserves 1 elastic IP, which is not listed below
+  micro:
+    instances: 1
+    dns_record: "micro"
+    ttl: 60
+  bat:
+    instances: 1
+    dns_record: "bat"
+    ttl: 60
+  bosh:
+    instances: 1
+    dns_record: bosh
+    ttl: 60
+
+key_pairs:
+  <%= key_pair_name %>: <%= key_pair_path %>
+
+<%- if has_package_cache_configuration? -%>
+compiled_package_cache:
+  access_key_id: <%= cache_access_key_id %>
+  secret_access_key: <%= cache_secret_access_key %>
+  bucket_name: <%= cache_bucket_name %>
+<%- end -%>
+<%- rds_sizes = production_resources? ? {:large => "db.m1.large", :huge => "db.m2.4xlarge"} : Hash.new("db.t1.micro") -%>
+rds:
+  - instance: ccdb
+    tag: cc
+    subnets:
+      - cf_rds1
+      - cf_rds2
+    aws_creation_options:
+      db_instance_class: <%= rds_sizes[:large] %>
+      # These are passed on directly to the AWS API call.
+      # http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/RDS/Client.html#create_db_instance-instance_method
+      <%- if production_resources? -%>
+      allocated_storage: 100
+      iops: 1000
+      <%- end -%>
+      multi_az: true
+  - instance: uaadb
+    tag: uaa
+    subnets:
+      - cf_rds1
+      - cf_rds2
+    aws_creation_options:
+      db_instance_class: <%= rds_sizes[:large] %>
+      <%- if production_resources? -%>
+      allocated_storage: 100
+      iops: 1000
+      <%- end -%>
+  - instance: mysql-service-public
+    tag: mysql
+    subnets:
+      - services_rds1
+      - services_rds2
+    aws_creation_options:
+      db_name: mgmt
+      engine_version: 5.5.27
+      db_instance_class: <%= rds_sizes[:huge] %>
+      <%- if production_resources? -%>
+      allocated_storage: 100
+      iops: 1000
+      <%- end -%>
+  - instance: bosh
+    tag: bosh
+    subnets:
+      - bosh_rds1
+      - bosh_rds2
+    aws_creation_options:
+      db_instance_class: <%= rds_sizes[:large] %>
+      multi_az: true

data/templates/aws_migration.erb

@@ -0,0 +1,5 @@
+class <%= class_name %> < Bosh::Aws::Migration
+  def execute
+
+  end
+end

data/templates/aws_migration_spec.erb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+require '<%= file_prefix %>'
+
+describe <%= class_name %> do
+  include MigrationSpecHelper
+
+  subject { described_class.new(config, '')}
+
+  it "migrates your cloud" do
+    expect { subject.execute }.to_not raise_error
+  end
+end

data/templates/bat.yml.erb

@@ -0,0 +1,84 @@
+---
+name: <%= deployment_name %>
+director_uuid: <%= director_uuid %>
+cpi: aws
+
+release:
+  name: bat
+  version: latest
+
+resource_pools:
+- name: default
+  stemcell:
+    name: <%= stemcell_name %>
+    version: <%= stemcell_version %>
+  network: default
+  size: 1
+  cloud_properties:
+    instance_type: m1.small
+    availability_zone: <%= availability_zone %>
+
+compilation:
+  reuse_compilation_vms: true
+  workers: 8
+  network: default
+  cloud_properties:
+    instance_type: c1.medium
+    availability_zone: <%= availability_zone %>
+
+update:
+  canaries: 1
+  canary_watch_time: 3000-90000
+  update_watch_time: 3000-90000
+  max_in_flight: 1
+  max_errors: 1
+
+networks:
+
+- name: default
+  type: manual
+  subnets:
+  - range: 10.10.0.0/24
+    reserved:
+    - 10.10.0.2 - 10.10.0.9
+    static:
+    - 10.10.0.10 - 10.10.0.30
+    gateway: 10.10.0.1
+    security_groups:
+    - bat
+    cloud_properties:
+      security_groups: bat
+      subnet: <%= subnet %>
+
+jobs:
+- name: "batlight"
+  template: "batlight"
+  instances: 1
+  resource_pool: default
+  networks:
+  - name: default
+    default: [dns, gateway]
+
+properties:
+  static_ip: <%= vip %>
+  uuid: <%= director_uuid %>
+  pool_size: 1
+  stemcell:
+    name: <%= stemcell_name %>
+    version: <%= stemcell_version %>
+  instances: 1
+  key_name:  <%= key_pair_name %>
+  mbus: nats://nats:0b450ada9f830085e2cdeff6@micro.<%= domain %>:4222
+  network:
+    cidr: 10.10.0.0/24
+    reserved:
+    - 10.10.0.2 - 10.10.0.9
+    static:
+    - 10.10.0.10 - 10.10.0.30
+    gateway: 10.10.0.1
+    subnet: <%= subnet %>
+    security_groups:
+    - bat
+  batlight:
+    missing: nope
+

data/templates/bosh.yml.erb

@@ -0,0 +1,198 @@
+---
+name: <%= bosh_deployment_name %>
+director_uuid: <%= director_uuid %>
+
+release:
+  name: bosh
+  version: latest
+
+networks:
+- name: default
+  type: manual
+  subnets:
+  - range: 10.10.0.0/24
+    gateway: 10.10.0.1
+    static:
+    - 10.10.0.7 - 10.10.0.9
+    reserved:
+    - 10.10.0.2 - 10.10.0.6
+    - 10.10.0.10 - 10.10.0.10
+    dns:
+    - 10.10.0.6
+    cloud_properties:
+      subnet: <%= subnet %>
+- name: vip_network
+  type: vip
+  # Fake network properties to satisfy bosh diff
+  subnets:
+  - range: 127.0.99.0/24
+    gateway: 127.0.99.1
+    dns:
+    - 127.0.99.250
+  cloud_properties:
+    security_groups:
+    - bosh
+
+resource_pools:
+- name: default
+  stemcell:
+    name: <%= stemcell_name %>
+    version: latest
+  network: default
+  size: 1
+  cloud_properties:
+    instance_type: m1.small
+    availability_zone: <%= availability_zone %>
+
+compilation:
+  reuse_compilation_vms: true
+  workers: 8
+  network: default
+  cloud_properties:
+    instance_type: c1.medium
+    availability_zone: <%= availability_zone %>
+
+
+update:
+  canaries: 1
+  canary_watch_time: 30000 - 90000
+  update_watch_time: 30000 - 90000
+  max_in_flight: 1
+  max_errors: 1
+
+jobs:
+- name: bosh
+  template:
+  - nats
+  - blobstore
+  - redis
+  - powerdns
+  - director
+  - registry
+  - health_monitor
+  instances: 1
+  resource_pool: default
+  persistent_disk: 20480
+  networks:
+  - name: default
+    default: [dns, gateway]
+    static_ips:
+    - 10.10.0.7
+  - name: vip_network
+    static_ips:
+    - <%= vip %>
+
+properties:
+  template_only:
+    aws:
+      availability_zone: <%= availability_zone %>
+
+  ntp:
+  - 0.north-america.pool.ntp.org
+  - 1.north-america.pool.ntp.org
+  - 2.north-america.pool.ntp.org
+  - 3.north-america.pool.ntp.org
+
+  blobstore:
+    address: 10.10.0.7
+    port: 25251
+    backend_port: 25552
+    agent:
+      user: agent
+      password: ldsjlkadsfjlj
+    director:
+      user: director
+      password: DirectoR
+
+  networks:
+    apps: default
+    management: default
+
+  nats:
+    user: nats
+    password: 0b450ada9f830085e2cdeff6
+    address: 10.10.0.7
+    port: 4222
+
+  mysql: &bosh_db
+    adapter: mysql2
+    user: <%= bosh_rds_user %>
+    password: <%= bosh_rds_password %>
+    host: <%= bosh_rds_host %>
+    port: <%= bosh_rds_port %>
+    database: bosh
+
+  redis:
+    address: 127.0.0.1
+    port: 25255
+    password: R3d!S
+
+  director:
+    name: <%= bosh_deployment_name %>
+    address: 10.10.0.7
+    port: 25555
+    encryption: false
+    enable_snapshots: true
+    db: *bosh_db
+    ssl:
+      key: |
+        <%= director_ssl_key %>
+      cert: |
+        <%= director_ssl_cert %>
+
+  hm:
+    http:
+      port: 25923
+      user: admin
+      password: admin
+    director_account:
+      user: <%= hm_director_user %>
+      password: <%= hm_director_password %>
+    intervals:
+      poll_director: 60
+      poll_grace_period: 30
+      log_stats: 300
+      analyze_agents: 60
+      agent_timeout: 180
+      rogue_agent_alert: 180
+    loglevel: info
+    email_notifications: false
+    tsdb_enabled: false
+    cloud_watch_enabled: true
+    resurrector_enabled: true
+    <% if ENV['BOSH_DATADOG_API_KEY'] && ENV['BOSH_DATADOG_APP_KEY'] %>
+    datadog_enabled: true
+    datadog:
+      api_key: <%= ENV['BOSH_DATADOG_API_KEY'] %>
+      application_key: <%= ENV['BOSH_DATADOG_APP_KEY'] %>
+    <% end %>
+
+  registry:
+    address: 10.10.0.7
+    db: *bosh_db
+    http:
+      port: 25777
+      user: awsreg
+      password: awsreg
+
+  aws:
+    access_key_id: <%= access_key_id %>
+    secret_access_key: <%= secret_access_key %>
+    region: <%= region %>
+    default_key_name: <%= key_pair_name %>
+    ec2_endpoint: ec2.<%= region %>.amazonaws.com
+    default_security_groups: ["bosh"]
+
+  dns:
+    address: 10.10.0.7
+    db: *bosh_db
+    recursor: 208.67.220.220
+
+  <% if compiled_package_cache? %>
+  compiled_package_cache:
+    provider: s3
+    options:
+      access_key_id: <%= cache_access_key_id %>
+      secret_access_key: <%= cache_secret_access_key %>
+      bucket_name: <%= cache_bucket_name %>
+   <% end %>

data/templates/micro_bosh.yml.erb

@@ -0,0 +1,82 @@
+---
+name: micro-<%= name %>
+
+logging:
+  level: DEBUG
+
+network:
+  type: manual
+  vip:  <%= vip %>
+  ip: 10.10.0.6
+  dns:
+  - 10.10.0.2
+  cloud_properties:
+    subnet: <%= subnet %>
+
+resources:
+  persistent_disk: 20000
+  cloud_properties:
+    instance_type: m1.small
+    availability_zone: <%= availability_zone %>
+
+cloud:
+  plugin: aws
+  properties:
+    aws:
+      access_key_id: <%= access_key_id %>
+      secret_access_key: <%= secret_access_key %>
+      default_key_name: <%= key_pair_name %>
+      default_security_groups: ["bosh"]
+      region: <%= region %>
+      ec2_private_key: <%= private_key_path %>
+    agent:
+      ntp:
+      - 0.north-america.pool.ntp.org
+      - 1.north-america.pool.ntp.org
+      - 2.north-america.pool.ntp.org
+      - 3.north-america.pool.ntp.org
+
+apply_spec:
+  agent:
+    blobstore:
+      address: 10.10.0.6
+    nats:
+      address: 10.10.0.6
+  properties:
+    ntp:
+    - 0.north-america.pool.ntp.org
+    - 1.north-america.pool.ntp.org
+    - 2.north-america.pool.ntp.org
+    - 3.north-america.pool.ntp.org
+    registry:
+      address: 10.10.0.6
+    dns:
+      recursor: 208.67.220.220
+    hm:
+      resurrector_enabled: true
+      director_account:
+        user: <%= hm_director_user %>
+        password: <%= hm_director_password %>
+    aws:
+      access_key_id: <%= access_key_id %>
+      secret_access_key: <%= secret_access_key %>
+      default_key_name: <%= key_pair_name %>
+      default_security_groups: ["bosh"]
+      ec2_endpoint: ec2.<%= region %>.amazonaws.com
+      region: <%= region %>
+    <% if compiled_package_cache? %>
+    compiled_package_cache:
+      provider: s3
+      options:
+        access_key_id: <%= cache_access_key_id %>
+        secret_access_key: <%= cache_secret_access_key %>
+        bucket_name: <%= cache_bucket_name %>
+    <% end %>
+    director:
+      enable_snapshots: true
+      ssl:
+        key: |
+          <%= director_ssl_key %>
+        cert: |
+          <%= director_ssl_cert %>
+