bosh-cloudfoundry 0.2.1 → 0.3.0

data/ChangeLog.md

@@ -0,0 +1,11 @@
+# Change Log
+
+## v0.3
+
+Additions:
+
+* OpenStack support! [thx @frodenas; tested by @danhighman] many hugs & kisses to you both!
+
+Fixes:
+
+* changed `health_manager` to `health_manager_next` job (legacy hm was removed in [cf-release patch](https://github.com/cloudfoundry/cf-release/commit/cba60f2e2dee13b7e09eb178eec72aa084a15b1a))

data/README.md

@@ -18,7 +18,16 @@ bosh-bootstrap deploy --latest-stemcell
 bosh-bootstrap ssh
 
 # now on the inception VM
-sudo gem install bosh-cloudfoundry
+
+# TODO - restore gem installation when rubygem security issue resolved
+# sudo gem install bosh-cloudfoundry
+
+cd /var/vcap/store/repos
+git clone git://github.com/StarkAndWayne/bosh-cloudfoundry.git
+cd bosh-cloudfoundry
+git checkout -b v0.3
+git pull origin v0.3
+rake install
 
 export TMPDIR=/var/vcap/store/tmp
 bosh cf upload release --dev
@@ -31,14 +40,14 @@ bosh cf prepare system production --release-name appcloud-dev
 bosh cf prepare system production
 
 bosh cf upload stemcell --latest
-bosh cf merge gerrit 37/13137/4 84/13084/4 83/13583/1
-bosh deploy
+bosh cf merge gerrit 37/13137/4 84/13084/4 83/13583/1 09/13609/2
+bosh cf deploy
 
 # now we can grow our single VM deployment
 
 bosh cf change deas 1
 bosh cf add service postgresql
-bosh deploy
+bosh cf deploy
 ```
 
 Overtime, as you add more DEAs and other service nodes, your set of VMs might look like:
@@ -87,7 +96,7 @@ bosh cf prepare system production
 # prompts for a DNS host for your CloudFoundry, such as mycompany.com
 bosh cf change deas 1
 bosh cf add service postgresql 1
-bosh deploy
+bosh cf deploy
 ```
 
 During `bosh cf prepare system production`, it will automatically upload the latest release of CloudFoundry (the latest final [BOSH release](http://github.com/cloudfoundry/cf-release)) and the latest stable stemcell (becomes the base AMI for AWS, for example).

data/lib/bosh-cloudfoundry/config/common_config.rb

@@ -1,5 +1,7 @@
 # Copyright (c) 2012-2013 Stark & Wayne, LLC
 
+require "cli"
+
 module Bosh; module CloudFoundry; module Config; end; end; end
 
 module Bosh::CloudFoundry::Config

data/lib/bosh-cloudfoundry/config/dea_config.rb

@@ -121,11 +121,14 @@ class Bosh::CloudFoundry::Config::DeaConfig
     }
   end
 
+  # The RAM for a dedicated DEA node
+  # else the RAM of the core/0 VM
+  # minus the +preallocated_ram+.
   def max_memory
     if dea_server_count > 0
       max_memory_for_dedicated_dea
     else
-      512
+      dea_ram_for_core_vm_flavor
     end
   end
 
@@ -139,6 +142,14 @@ class Bosh::CloudFoundry::Config::DeaConfig
     300
   end
 
+  def dea_ram_for_core_vm_flavor
+    ram_for_core_vm_flavor - preallocated_ram
+  end
+
+  def ram_for_core_vm_flavor
+    provider.ram_for_server_flavor(system_config.core_server_flavor)
+  end
+
   def ram_for_server_flavor
     provider.ram_for_server_flavor(dea_server_flavor)
   end

data/lib/bosh-cloudfoundry/config/microbosh_config.rb

@@ -38,6 +38,15 @@ class Bosh::CloudFoundry::Config::MicroboshConfig
         aws_access_key_id: provider_credentials["access_key_id"],
         aws_secret_access_key: provider_credentials["secret_access_key"]
       }
+    elsif openstack?
+      {
+        provider: "openstack",
+        openstack_username: provider_credentials["username"],
+        openstack_api_key: provider_credentials["api_key"],
+        openstack_tenant: provider_credentials["tenant"],
+        openstack_auth_url: provider_credentials["auth_url"],
+        openstack_region: provider_credentials["region"]
+      }
     else
       raise "please implement #fog_credentials for #{bosh_provider}"
     end
@@ -55,6 +64,10 @@ class Bosh::CloudFoundry::Config::MicroboshConfig
     bosh_provider == "aws"
   end
 
+  def openstack?
+    bosh_provider == "openstack"
+  end
+
   # micro_bosh.yml looks like:
   #
   # cloud:

data/lib/bosh-cloudfoundry/config/system_config.rb

@@ -40,7 +40,8 @@ class Bosh::CloudFoundry::Config::SystemConfig < Bosh::Cli::Config
     :dea,              # e.g. { "count" => 2, "flavor" => "m1.large" }
     :postgresql,       # e.g. [{ "count" => 2, "flavor" => "m1.large", "plan" => "free" }]
     :redis,            # e.g. [{ "count" => 2, "flavor" => "m1.large", "plan" => "free" }]
-    :security_group    # e.g. "cloudfoundry-production"
+    :security_group,   # e.g. "cloudfoundry-production"
+    :system_initialized,  # e.g. true / false
   ].each do |attr|
     define_method attr do
       read(attr, false)

data/lib/bosh-cloudfoundry/config_options.rb

@@ -46,7 +46,7 @@ module Bosh::CloudFoundry::ConfigOptions
       system_config = Bosh::CloudFoundry::Config::SystemConfig.new(system)
       system_config.bosh_target = options[:bosh_target] || config.target
       system_config.bosh_target_uuid = options[:bosh_target_uuid] || config.target_uuid
-      system_config.bosh_provider = 'aws' # TODO support other BOSH providers
+      system_config.bosh_provider = options[:bosh_provider] || bosh_cpi
       system_config.release_name ||= DEFAULT_RELEASE_NAME
       system_config.release_version ||= DEFAULT_RELEASE_VERSION
       system_config.stemcell_name ||= DEFAULT_STEMCELL_NAME
@@ -196,6 +196,10 @@ module Bosh::CloudFoundry::ConfigOptions
     options[:bosh_git_repo] || common_config.bosh_git_repo
   end
 
+  def bosh_cpi
+    `bosh status | grep CPI | awk '{ print $2 }'`.strip
+  end
+
   def deployment_manifest(subsystem="core")
     YAML.load_file(deployment_manifest_path(subsystem))
   end
@@ -223,6 +227,15 @@ module Bosh::CloudFoundry::ConfigOptions
     end
   end
 
+  def system_initialized?
+    system_config.system_initialized
+  end
+
+  def system_initialized!
+    system_config.system_initialized = true
+    system_config.save
+  end
+
   def choose_releases_dir
     if non_interactive?
       err "Please set releases_dir configuration for non-interactive mode"
@@ -280,13 +293,13 @@ module Bosh::CloudFoundry::ConfigOptions
       err "Please set core_ip configuration for non-interactive mode"
     end
 
-    if aws?
+    if aws? || openstack?
       system_config.core_ip = ask("Main public IP address (press Enter to provision new IP): ").to_s
     else
       system_config.core_ip = ask("Main public IP address: ").to_s
     end
     if system_config.core_ip.blank?
-      say "Provisioning #{bosh_provider} public IP address..."
+      say "Provisioning #{system_config.bosh_provider} public IP address..."
       system_config.core_ip = provider.provision_public_ip_address
       if system_config.core_ip.blank?
         say "Hmmm, I wasn't able to get a public IP at the moment. Perhaps try again or provision it manually?".red

data/lib/bosh-cloudfoundry/providers.rb

@@ -10,6 +10,8 @@ module Bosh::CloudFoundry::Providers
     case system_config.bosh_provider.to_sym
     when :aws
       Bosh::CloudFoundry::Providers::AWS.new(system_config.microbosh.fog_compute)
+    when :openstack
+      Bosh::CloudFoundry::Providers::OpenStack.new(system_config.microbosh.fog_compute)
     else
       raise "please support #{system_config.bosh_provider} provider"
     end
@@ -17,3 +19,4 @@ module Bosh::CloudFoundry::Providers
 end
 
 require "bosh-cloudfoundry/providers/aws"
+require "bosh-cloudfoundry/providers/openstack"

data/lib/bosh-cloudfoundry/providers/openstack.rb

@@ -0,0 +1,69 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Bosh; module CloudFoundry; module Providers; end; end; end
+
+class Bosh::CloudFoundry::Providers::OpenStack
+  attr_reader :fog_compute
+  def initialize(fog_compute=nil)
+    @fog_compute = fog_compute
+  end
+
+  # @return [String] provisions a new public IP address in target region
+  # TODO nil if none available
+  def provision_public_ip_address
+    address = fog_compute.addresses.create
+    address.ip
+    # TODO catch error and return nil
+  end
+
+  # @return [Integer] megabytes of RAM for requested flavor of server
+  def ram_for_server_flavor(server_flavor)
+    if flavor = fog_compute_flavor(server_flavor)
+      flavor.ram
+    else
+      raise "Unknown OpenStack flavor '#{server_flavor}'"
+    end
+  end
+
+  def fog_compute_flavor(server_flavor)
+    compute_flavors.find { |f| f.name == server_flavor }
+  end
+
+  def compute_flavors
+    fog_compute.flavors
+  end
+
+  # Creates or reuses an OpenStack security group and opens ports.
+  #
+  # +security_group_name+ is the name to be created or reused
+  # +ports+ is a hash of name/port for ports to open, for example:
+  # {
+  #   ssh: 22,
+  #   http: 80,
+  #   https: 443
+  # }
+  def create_security_group(security_group_name, ports)
+    security_groups = fog_compute.security_groups
+    unless sg = security_groups.find { |s| s.name == security_group_name }
+      sg = fog_compute.security_groups.create(name: security_group_name, description: security_group_name)
+      puts "Created security group #{security_group_name}"
+    else
+      puts "Reusing security group #{security_group_name}"
+    end
+    ip_permissions = sg.rules
+    ports_opened = 0
+    ports.each do |name, port|
+      unless port_open?(ip_permissions, port)
+        sg.create_security_group_rule(port, port)
+        puts " -> opened #{name} port #{port}"
+        ports_opened += 1
+      end
+    end
+    puts " -> no additional ports opened" if ports_opened == 0
+    true
+  end
+
+  def port_open?(ip_permissions, port)
+    ip_permissions && ip_permissions.find {|ip| ip["from_port"] <= port && ip["to_port"] >= port }
+  end
+end

data/lib/bosh-cloudfoundry/system_deployment_manifest_renderer.rb

@@ -111,6 +111,8 @@ class Bosh::CloudFoundry::SystemDeploymentManifestRenderer
   def cloud_properties_for_server_flavor(server_flavor)
     if aws?
       { "instance_type" => server_flavor }
+    elsif openstack?
+      { "instance_type" => server_flavor }
     else
       raise 'Please implement #{self.class}#cloud_properties_for_server_flavor'
     end
@@ -120,6 +122,10 @@ class Bosh::CloudFoundry::SystemDeploymentManifestRenderer
     system_config.bosh_provider == "aws"
   end
 
+  def openstack?
+    system_config.bosh_provider == "openstack"
+  end
+
   # 
   def base_manifest(
       deployment_name,
@@ -179,7 +185,7 @@ class Bosh::CloudFoundry::SystemDeploymentManifestRenderer
          ["postgres",
           "nats",
           "router",
-          "health_manager",
+          "health_manager_next",
           "cloud_controller",
           # "debian_nfs_server",
           # "serialization_data_server",
@@ -280,7 +286,7 @@ class Bosh::CloudFoundry::SystemDeploymentManifestRenderer
          "password"=>common_password,
          "maxmemory"=>500000000},
        "service_plans"=>{},
-       "dea"=>{"max_memory"=>512}}}
+       "dea"=>{"max_memory"=>4096}}}
   end
 
 end

data/lib/bosh-cloudfoundry/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Cloudfoundry
-    VERSION = "0.2.1"
+    VERSION = "0.3.0"
   end
 end

data/lib/bosh/cli/commands/cf.rb

@@ -164,6 +164,16 @@ module Bosh::Cli::Command
         set_deployment(deployment)
         bosh_cmd "deploy"
       end
+      email = system_config.admin_emails.first
+      password = system_config.common_password
+      sh "sudo gem install vmc --no-ri --no-rdoc" unless system_initialized?
+      sh "vmc target http://api.#{root_dns}"
+      if system_initialized?
+        sh "vmc login #{email} --password #{password}"
+      else
+        sh "vmc register #{email} --password #{password} --verify #{password}"
+      end
+      system_initialized!
     end
 
     usage "cf watch nats"
@@ -248,6 +258,8 @@ module Bosh::Cli::Command
     def bosh_provider
       if aws?
         "aws"
+      elsif openstack?
+        "openstack"
       else
         err("Please implement cf.rb's bosh_provider for this IaaS")
       end
@@ -255,9 +267,14 @@ module Bosh::Cli::Command
 
     # Deploying CloudFoundry to AWS?
     # Is the target BOSH's IaaS using the AWS CPI?
-    # FIXME Currently only AWS is supported so its always AWS
     def aws?
-      true
+      system_config.bosh_provider == "aws"
+    end
+
+    # Deploying CloudFoundry to OpenStack?
+    # Is the target BOSH's IaaS using the OpenStack CPI?
+    def openstack?
+      system_config.bosh_provider == "openstack"
     end
 
     # User is prompted for common values at the
@@ -437,7 +454,7 @@ module Bosh::Cli::Command
     # | bosh-stemcell-aws-0.6.7.tgz             | aws                    |
     def bosh_stemcell_name(stemcell_type)
       tags = [bosh_provider]
-      tags << "stable" if stemcell_type == "stable"
+      tags << "stable" if stemcell_type == "stable" unless openstack?
       bosh_stemcells_cmd = "bosh public stemcells --tags #{tags.join(',')}"
       say "Locating bosh stemcell, running '#{bosh_stemcells_cmd}'..."
       `#{bosh_stemcells_cmd} | grep ' bosh-stemcell-' | awk '{ print $2 }' | sort -r | head -n 1`.strip
@@ -528,6 +545,10 @@ module Bosh::Cli::Command
         unless aws_compute_flavors.select { |flavor| flavor[:id] == flavor }
           err("Server flavor '#{flavor}' is not a valid AWS compute flavor")
         end
+      elsif openstack?
+        unless provider.fog_compute_flavor(flavor)
+          err("Server flavor '#{flavor}' is not a valid OpenStack compute flavor")
+        end
       else
         err("Please implemenet cf.rb's validate_compute_flavor for this IaaS")
       end
@@ -537,9 +558,7 @@ module Bosh::Cli::Command
     # then ensure that a generated value is stored
     def generate_generatable_options
       common_password
-      if aws?
-        security_group
-      end
+      security_group
     end
 
     # Renders the +SystemConfig+ model (+system_config+) into the system's
@@ -626,6 +645,8 @@ module Bosh::Cli::Command
     def default_core_server_flavor
       if aws?
         "m1.large"
+      elsif openstack?
+        "m1.large"
       else
         err("Please implement cf.rb's default_core_server_flavor for this IaaS")
       end
@@ -634,6 +655,8 @@ module Bosh::Cli::Command
     def default_dea_server_flavor
       if aws?
         "m1.large"
+      elsif openstack?
+        "m1.large"
       else
         err("Please implement cf.rb's default_server_flavor for this IaaS")
       end
@@ -642,6 +665,8 @@ module Bosh::Cli::Command
     def default_service_server_flavor(service_name)
       if aws?
         "m1.xlarge"
+      elsif openstack?
+        "m1.xlarge"
       else
         err("Please implement cf.rb's default_service_server_flavor for this IaaS")
       end

data/spec/assets/deployments/aws-core-1-m1.small-free-redis.yml

@@ -3,7 +3,7 @@ name: production-core
 director_uuid: DIRECTOR_UUID
 release: 
   name: appcloud
-  version: 124
+  version: latest
 compilation: 
   workers: 10
   network: default
@@ -52,7 +52,7 @@ jobs:
   - postgres
   - nats
   - router
-  - health_manager
+  - health_manager_next
   - cloud_controller
   - stager
   - uaa
@@ -207,7 +207,7 @@ properties:
           backup: 
             enable: true
   dea: 
-    max_memory: 512
+    max_memory: 1440.8
   redis_gateway: 
     token: TOKEN
     supported_versions: 

data/spec/assets/deployments/aws-core-1-m1.xlarge-free-postgresql-2-m1.small-free-postgresql.yml

@@ -3,7 +3,7 @@ name: production-core
 director_uuid: DIRECTOR_UUID
 release: 
   name: appcloud
-  version: 124
+  version: latest
 compilation: 
   workers: 10
   network: default
@@ -61,7 +61,7 @@ jobs:
   - postgres
   - nats
   - router
-  - health_manager
+  - health_manager_next
   - cloud_controller
   - stager
   - uaa
@@ -228,7 +228,7 @@ properties:
           backup: 
             enable: true
   dea: 
-    max_memory: 512
+    max_memory: 1440.8
   postgresql_gateway: 
     admin_user: psql_admin
     admin_passwd_hash: 

data/spec/assets/deployments/aws-core-2-m1.xlarge-dea.yml

@@ -3,7 +3,7 @@ name: production-core
 director_uuid: DIRECTOR_UUID
 release: 
   name: appcloud
-  version: 124
+  version: latest
 compilation: 
   workers: 10
   network: default
@@ -51,7 +51,7 @@ jobs:
   - postgres
   - nats
   - router
-  - health_manager
+  - health_manager_next
   - cloud_controller
   - stager
   - uaa

data/spec/assets/deployments/aws-core-only.yml

@@ -3,7 +3,7 @@ name: production-core
 director_uuid: DIRECTOR_UUID
 release: 
   name: appcloud
-  version: 124
+  version: latest
 compilation: 
   workers: 10
   network: default
@@ -43,7 +43,7 @@ jobs:
   - postgres
   - nats
   - router
-  - health_manager
+  - health_manager_next
   - cloud_controller
   - stager
   - uaa
@@ -175,4 +175,4 @@ properties:
   service_plans: {}
 
   dea: 
-    max_memory: 512
+    max_memory: 1440.8

data/spec/assets/deployments/aws-core-with-nfs.yml

@@ -43,7 +43,7 @@ jobs:
   - postgres
   - nats
   - router
-  - health_manager
+  - health_manager_next
   - cloud_controller
   - debian_nfs_server
   - serialization_data_server

data/spec/unit/cf_command_spec.rb

@@ -247,6 +247,10 @@ describe Bosh::Cli::Command::Base do
       end
       @cmd.should_receive(:set_deployment).exactly(2).times
       @cmd.should_receive(:sh).with("bosh -n --color deploy").exactly(2).times
+      @cmd.should_receive(:sh).with("sudo gem install vmc --no-ri --no-rdoc")
+      @cmd.should_receive(:sh).with("vmc target http://api.mycompany.com")
+      @cmd.should_receive(:sh).with(
+        "vmc register drnic@starkandwayne.com --password c1oudc0wc1oudc0w --verify c1oudc0wc1oudc0w")
       @cmd.deploy
     end
 

data/spec/unit/config/dea_config_spec.rb

@@ -12,7 +12,10 @@ describe Bosh::CloudFoundry::Config::DeaConfig do
     @system_dir = File.join(@systems_dir, "production")
     mkdir_p(@system_dir)
     @system_config = Bosh::CloudFoundry::Config::SystemConfig.new(@system_dir)
+    @system_config.bosh_target = "http://6.7.8.9:25555"
+    @system_config.bosh_target_uuid = "DIRECTOR_UUID"
     @system_config.bosh_provider = "aws"
+    @system_config.core_server_flavor = "m1.large"
     @manifest = YAML.load_file(spec_asset("deployments/aws-core-only.yml"))
   end
 
@@ -48,7 +51,7 @@ describe Bosh::CloudFoundry::Config::DeaConfig do
       subject.merge_manifest_properties(@manifest)
       @manifest["properties"]["dea"].should_not be_nil
       @manifest["properties"]["dea"]["max_memory"].should_not be_nil
-      @manifest["properties"]["dea"]["max_memory"].should == 512
+      @manifest["properties"]["dea"]["max_memory"].should == 7380
     end
   end
   describe "5 x m1.xlarge deas on AWS" do

data/spec/unit/system_deployment_manifest_renderer_spec.rb

@@ -19,9 +19,11 @@ describe Bosh::CloudFoundry::SystemDeploymentManifestRenderer do
     @system_dir = File.join(@systems_dir, "production")
     mkdir_p(@system_dir)
     @system_config = Bosh::CloudFoundry::Config::SystemConfig.new(@system_dir)
+    @system_config.bosh_target = "http://6.7.8.9:25555"
+    @system_config.bosh_target_uuid = "DIRECTOR_UUID"
     @system_config.bosh_provider = 'aws'
     @system_config.release_name = 'appcloud'
-    @system_config.release_version = 124 # TODO restore to 'latest' when #49 fixed
+    @system_config.release_version = 'latest'
     @system_config.stemcell_name = 'bosh-stemcell'
     @system_config.stemcell_version = '0.6.4'
     @system_config.core_server_flavor = 'm1.small'
@@ -34,6 +36,12 @@ describe Bosh::CloudFoundry::SystemDeploymentManifestRenderer do
 
     @renderer = Bosh::CloudFoundry::SystemDeploymentManifestRenderer.new(
       @system_config, @common_config, @bosh_config)
+
+    Bosh::CloudFoundry::Providers.should_receive(:for_bosh_provider_name).
+      and_return(Bosh::CloudFoundry::Providers::AWS.new)
+    # 
+    # @dea_config = @renderer.dea_config
+    # @dea_config.should_receive(:ram_for_server_flavor).and_return(1700)
   end
 
   after(:each) do
@@ -51,8 +59,6 @@ describe Bosh::CloudFoundry::SystemDeploymentManifestRenderer do
       end
     end
     it "renders a simple system + DEAs into a deployment manifest" do
-      Bosh::CloudFoundry::Providers.should_receive(:for_bosh_provider_name).
-        and_return(Bosh::CloudFoundry::Providers::AWS.new)
       @system_config.dea = { "count" => 2, "flavor" => "m1.xlarge" }
       @renderer.perform
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh-cloudfoundry
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-27 00:00:00.000000000 Z
+date: 2013-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_cli
@@ -149,6 +149,7 @@ files:
 - .gitignore
 - .rspec
 - .travis.yml
+- ChangeLog.md
 - Gemfile
 - Guardfile
 - LICENSE.txt
@@ -169,6 +170,7 @@ files:
 - lib/bosh-cloudfoundry/gerrit_patches_helper.rb
 - lib/bosh-cloudfoundry/providers.rb
 - lib/bosh-cloudfoundry/providers/aws.rb
+- lib/bosh-cloudfoundry/providers/openstack.rb
 - lib/bosh-cloudfoundry/system_deployment_manifest_renderer.rb
 - lib/bosh-cloudfoundry/version.rb
 - lib/bosh/cli/commands/cf.rb
@@ -207,7 +209,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4073595004324325457
+      hash: -743491069374654137
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -216,7 +218,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4073595004324325457
+      hash: -743491069374654137
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24