bosh-cloudfoundry 0.7.0.alpha.3 → 0.7.0.alpha.4

data/ChangeLog.md

@@ -17,5 +17,6 @@ The rewrite introduces some new implementation/feature concepts:
 * using `bosh diff` (aka biff) to generate the deployment file
 * bundles all final releases into the project & distributed rubygem/plugin (no runtime dependency on cf-release git repository; only the public blobstore)
 * templates are versioned for the final releases
+* different sizes of deployments (orders of magnitude), such as small, medium & large
 
 The latter means that new versions of this rubygem can be published that are backwards compatible with aging deployments of Cloud Foundry. There should not be any forced coupling of old `bosh-cloudfoundry` to old `cf-release` final releases.

data/bosh-cloudfoundry.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "bosh-cloudfoundry"
-  spec.version       = "0.7.0.alpha.3"
+  spec.version       = "0.7.0.alpha.4"
   spec.authors       = ["Dr Nic Williams"]
   spec.email         = ["drnicwilliams@gmail.com"]
   spec.description   = %q{Create & manage Cloud Foundry deployments}
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rspec", "~> 2.13.0"
+  spec.add_development_dependency "fakeweb"
 end

data/lib/bosh/cli/commands/cf.rb

@@ -1,11 +1,11 @@
 require "yaml"
-
 require "bosh/cloudfoundry"
 
 module Bosh::Cli::Command
   class CloudFoundry < Base
     include FileUtils
     include Bosh::Cli::Validation
+    include Bosh::Cloudfoundry
 
     usage "cf"
     desc  "show micro cf sub-commands"
@@ -23,6 +23,7 @@ module Bosh::Cli::Command
     def prepare_cf
       auth_required
 
+      release_yml = Dir[File.join(bosh_release_dir, "releases", "*-#{latest_release_version}.yml")].first
       release_cmd(non_interactive: true).upload(release_yml)
     end
 
@@ -33,7 +34,13 @@ module Bosh::Cli::Command
     option "--name cf-<timestamp>", "Unique bosh deployment name"
     option "--disk 4096", Integer, "Size of persistent disk (Mb)"
     option "--security-group default", String, "Security group to assign to provisioned VMs"
+    option "--deployment-size medium", String, "Size of deployment - medium or large"
     def create_cf
+      auth_required
+      bosh_status # preload
+
+      setup_deployment_attributes
+
       ip_addresses = options[:ip]
       err("USAGE: bosh create cf --ip 1.2.3.4 -- please provide one IP address that will be bound to router.") if ip_addresses.blank?
       err("Only one IP address is supported currently. Please create an issue to mention you need more.") if ip_addresses.size > 1
@@ -43,25 +50,27 @@ module Bosh::Cli::Command
       err("USAGE: bosh create cf --dns mycloud.com -- please provide a base DNS that has a '*' A record referencing IPs") unless dns
       attrs.set(:dns, dns)
 
-      auth_required
-      bosh_status # preload
-
       attrs.set_unless_nil(:name, options[:name])
-      attrs.set_unless_nil(:core_size, options[:core_size] || options[:size])
       attrs.set_unless_nil(:persistent_disk, options[:disk])
       attrs.set_unless_nil(:security_group, options[:security_group])
+      attrs.set_unless_nil(:common_password, options[:common_password])
+      attrs.set_unless_nil(:deployment_size, options[:deployment_size])
+
+      release_version = ReleaseVersion.latest_version_number
+      @release_version_cpi_size = 
+        ReleaseVersionCpiSize.new(@release_version_cpi, attrs.deployment_size)
 
       nl
       say "CPI: #{bosh_cpi.make_green}"
       say "DNS mapping: #{attrs.validated_color(:dns)} --> #{attrs.validated_color(:ip_addresses)}"
       say "Deployment name: #{attrs.validated_color(:name)}"
-      say "Resource size: #{attrs.validated_color(:core_size)}"
+      say "Deployment size: #{attrs.validated_color(:deployment_size)}"
       say "Persistent disk: #{attrs.validated_color(:persistent_disk)}"
       say "Security group: #{attrs.validated_color(:security_group)}"
       nl
 
-      step("Validating resource size", "Resource size must be in #{attrs.available_resource_sizes.join(', ')}", :non_fatal) do
-        attrs.validate(:core_size)
+      step("Validating deployment size", "Available deployment sizes are #{attrs.available_deployment_sizes.join(', ')}", :fatal) do
+        attrs.validate(:deployment_size)
       end
 
       unless confirmed?("Security group #{attrs.validated_color(:security_group)} exists with ports #{attrs.required_ports.join(", ")}")
@@ -73,53 +82,11 @@ module Bosh::Cli::Command
 
       raise Bosh::Cli::ValidationHalted unless errors.empty?
 
-      # Create an initial deployment file; upon which the CPI-specific template will be applied below
-      # Initial file will look like:
-      # ---
-      # name: NAME
-      # director_uuid: 4ae3a0f0-70a5-4c0d-95f2-7fafaefe8b9e
-      # releases:
-      #  - name: cf-release
-      #    version: 132
-      # networks: {}
-      # properties:
-      #   cf:
-      #     dns: mycloud.com
-      #     ip_addresses: ['1.2.3.4']
-      #     core_size: medium
-      #     security_group: cf
-      #     persistent_disk: 4096
-      step("Checking/creating #{deployment_file_dir} for deployment files",
-           "Failed to create #{deployment_file_dir} for deployment files", :fatal) do
-        mkdir_p(deployment_file_dir)
-      end
-
-      step("Creating deployment file #{deployment_file}",
-           "Failed to create deployment file #{deployment_file}", :fatal) do
-        File.open(deployment_file, "w") do |file|
-          file << {
-            "name" => attrs.name,
-            "director_uuid" => bosh_uuid,
-            "releases" => {
-              "name" => release_name,
-              "version" => release_version
-            },
-            "networks" => {},
-            "properties" => {
-              attrs.properties_key => attrs.attributes_with_string_keys
-            }
-          }.to_yaml
-        end
+      deployment_file = DeploymentFile.new(@release_version_cpi_size, attrs, bosh_status)
+      deployment_file.prepare_environment
+      deployment_file.create_deployment_file
+      deployment_file.deploy(options)
 
-        stdout = Bosh::Cli::Config.output
-        Bosh::Cli::Config.output = nil
-        deployment_cmd(non_interactive: true).set_current(deployment_file)
-        biff_cmd(non_interactive: true).biff(template_file)
-        Bosh::Cli::Config.output = stdout
-      end
-      # re-set current deployment to show output
-      deployment_cmd.set_current(deployment_file)
-      deployment_cmd(non_interactive: options[:non_interactive]).perform
     rescue Bosh::Cli::ValidationHalted
       errors.each do |error|
         say error.make_red
@@ -129,23 +96,28 @@ module Bosh::Cli::Command
     usage "show cf passwords"
     desc "display the internal passwords for deployment"
     def show_cf_passwords
-      load_deployment_into_attributes
+
+      setup_deployment_attributes
+      reconstruct_deployment_file
       say "Common password: #{attrs.validated_color(:common_password)}"
     end
 
     protected
-    def load_deployment_into_attributes
-      attrs.load_deployment_file(deployment)
+    def setup_deployment_attributes
+      @release_version_cpi = ReleaseVersionCpi.latest_for_cpi(bosh_cpi)
+      @deployment_attributes = DeploymentAttributes.new(director_client, bosh_status, @release_version_cpi)
     end
 
-    def release_versioned_template
-      @release_versioned_template ||= begin
-        Bosh::Cloudfoundry::ReleaseVersionedTemplate.new(release_version, bosh_cpi, deployment_size)
-      end
+    def attrs
+      @deployment_attributes
     end
 
-    def template_file
-      release_versioned_template.template_file_path
+    # After a deployment is created, the input properties/attributes are stored within the generated
+    # deployment file. Therefore, to update a deployment, first we must load in the attributes.
+    def reconstruct_deployment_file
+      @deployment_file = DeploymentFile.reconstruct_from_deployment_file(deployment, director_client, bosh_status)
+      @deployment_attributes = @deployment_file.deployment_attributes
+      @release_version_cpi_size = @deployment_file.release_version_cpi_size
     end
 
     def bosh_release_dir
@@ -173,73 +145,10 @@ module Bosh::Cli::Command
       latest_version
     end
 
-    def release_yml
-      @release_yml ||= begin
-        Dir[File.join(bosh_release_dir, "releases", "*-#{latest_release_version}.yml")].first
-      end
-    end
-
-    def attrs
-      @deployment_attributes ||= begin
-        klass = release_versioned_template.deployment_attributes_class
-        klass.new(director_client, bosh_status, release_versioned_template)
-      end
-    end
-
-    # TODO - determined by the release version (appcloud-131, cf-release-132, ...)
-    def release_name
-      "cf-release"
-    end
-
-    # TODO - support other release versions
-    def release_version
-      132
-    end
-
-    # TODO - support other deployment sizes
-    def deployment_size
-      "dev"
-    end
-
     def director_client
       director
     end
 
-    def deployment_file
-      File.join(deployment_file_dir, "#{attrs.name}.yml")
-    end
-
-    def deployment_file_dir
-      File.expand_path("deployments/cf")
-    end
-
-    def deployment_cmd(options = {})
-      cmd ||= Bosh::Cli::Command::Deployment.new
-      options.each do |key, value|
-        cmd.add_option key.to_sym, value
-      end
-      cmd
-    end
-
-    def release_cmd(options = {})
-      cmd ||= Bosh::Cli::Command::Release.new
-      options.each do |key, value|
-        cmd.add_option key.to_sym, value
-      end
-      cmd
-    end
-
-    def biff
-      @biff_cmd ||= Bosh::Cli::Command::Biff.new
-    end
-
-    def biff_cmd(options = {})
-      options.each do |key, value|
-        biff.add_option key.to_sym, value
-      end
-      biff
-    end
-
     def bosh_status
       @bosh_status ||= begin
         step("Fetching bosh information", "Cannot fetch bosh information", :fatal) do
@@ -249,10 +158,6 @@ module Bosh::Cli::Command
       end
     end
 
-    def bosh_uuid
-      bosh_status["uuid"]
-    end
-
     def bosh_cpi
       bosh_status["cpi"]
     end

data/lib/bosh/cloudfoundry.rb

@@ -1,4 +1,7 @@
 module Bosh; module Cloudfoundry; end; end
 
-require "bosh/cloudfoundry/release_versioned_template"
-require "bosh/cloudfoundry/deployment_attributes"
+require "bosh/cloudfoundry/release_version"
+require "bosh/cloudfoundry/release_version_cpi"
+require "bosh/cloudfoundry/release_version_cpi_size"
+require "bosh/cloudfoundry/deployment_attributes"
+require "bosh/cloudfoundry/deployment_file"

data/lib/bosh/cloudfoundry/deployment_attributes.rb

@@ -6,13 +6,15 @@ module Bosh::Cloudfoundry
     include Bosh::Cli::Validation
 
     attr_reader :attributes
-    def initialize(director_client, bosh_status, released_versioned_template, attributes = {})
+    attr_reader :released_version_cpi
+
+    def initialize(director_client, bosh_status, released_version_cpi, attributes = {})
       @director_client = director_client
       @bosh_status = bosh_status
-      @released_versioned_template = released_versioned_template
+      @released_version_cpi = released_version_cpi
       @attributes = attributes
       @attributes[:name] ||= default_name
-      @attributes[:core_size] ||= default_size
+      @attributes[:deployment_size] ||= default_deployment_size
       @attributes[:persistent_disk] ||= default_persistent_disk
       @attributes[:security_group] ||= default_security_group
       @attributes[:common_password] ||= random_string(12, :common)
@@ -22,8 +24,8 @@ module Bosh::Cloudfoundry
       @attributes[:name]
     end
 
-    def core_size
-      @attributes[:core_size]
+    def deployment_size
+      @attributes[:deployment_size]
     end
 
     def persistent_disk
@@ -35,11 +37,11 @@ module Bosh::Cloudfoundry
     end
 
     def ip_addresses
-      @attribute[:ip_addresses]
+      @attributes[:ip_addresses]
     end
 
     def dns
-      @attribute[:dns]
+      @attributes[:dns]
     end
 
     def set_unless_nil(attribute, value)
@@ -50,23 +52,12 @@ module Bosh::Cloudfoundry
       attributes[attribute.to_sym] = value if value
     end
 
-    def load_deployment_file(deployment_file)
-      deployment_obj = YAML.load_file(deployment_file)
-      attributes = deployment_obj["properties"][properties_key]
-      @attributes = attributes.inject({}) do |mem, key_value|
-        k, v = key_value; mem[k.to_sym] = v; mem
-      end
-    end
-
-    # attributes are stored within deployment file at properties.cf
-    def properties_key
-      "cf"
-    end
-
     def validate(attribute)
       value = attributes[attribute.to_sym]
-      if attribute.to_s =~ /size$/
-        available_resource_sizes.include?(value)
+      if attribute.to_s == "deployment_size"
+        available_deployment_sizes.include?(value)
+      elsif attribute.to_s =~ /size$/
+        available_resources.include?(value)
       else
         true
       end
@@ -83,12 +74,25 @@ module Bosh::Cloudfoundry
     end
 
     # TODO move these validations into a "ValidatedSize" class or similar
-    def available_resource_sizes
-      resources = @released_versioned_template.spec["resources"]
-      if resources && resources.is_a?(Array) && resources.first.is_a?(String)
-        resources
-      else
-        err "template spec needs 'resources' key with list of resource pool names available"
+    def available_resources
+      @available_resources ||= begin
+        resources = released_version_cpi.spec["resources"]
+        if resources && resources.is_a?(Array) && resources.first.is_a?(String)
+          resources
+        else
+          err "template spec needs 'resources' key with list of resource pool names available; found #{released_version_cpi.spec.inspect}"
+        end
+      end
+    end
+
+    def available_deployment_sizes
+      @available_deployment_sizes ||= begin
+        deployment_sizes = released_version_cpi.spec["deployment_sizes"]
+        if deployment_sizes && deployment_sizes.is_a?(Array) && deployment_sizes.first.is_a?(String)
+          deployment_sizes
+        else
+          err "template spec needs 'deployment_sizes' key with list of deployment sizes names available; found #{deployment_sizes.spec.inspect}"
+        end
       end
     end
 
@@ -114,8 +118,9 @@ module Bosh::Cloudfoundry
       "cf-#{Time.now.to_i}"
     end
 
-    def default_size
-      "small"
+    # TODO change to small when its implemented
+    def default_deployment_size
+      "medium"
     end
 
     def default_persistent_disk

data/lib/bosh/cloudfoundry/deployment_file.rb

@@ -0,0 +1,192 @@
+module Bosh::Cloudfoundry
+  # Prior to deploying (creating or updating or deleting) a bosh release we
+  # need a deployment file. This is created with this class +DeploymentFile+.
+  #
+  # The deployment file is a product of:
+  # * a release version/CPI/size (+ReleaseVersionCpiSize+) which provides a template; and
+  # * attributes (+DeploymentAttributes+) which populate the template
+  class DeploymentFile
+    include FileUtils
+    include Bosh::Cli::Validation
+
+    attr_reader :release_version_cpi_size
+    attr_reader :deployment_attributes
+    attr_reader :bosh_status
+
+    def initialize(release_version_cpi_size, deployment_attributes, bosh_status)
+      @release_version_cpi_size = release_version_cpi_size
+      @deployment_attributes = deployment_attributes
+      @bosh_status = bosh_status
+    end
+
+    def prepare_environment
+      step("Checking/creating #{deployment_file_dir} for deployment files",
+           "Failed to create #{deployment_file_dir} for deployment files", :fatal) do
+        mkdir_p(deployment_file_dir)
+      end
+    end
+
+    # Create an initial deployment file; upon which the CPI-specific template will be applied below
+    # Initial file will look like:
+    # ---
+    # name: NAME
+    # director_uuid: 4ae3a0f0-70a5-4c0d-95f2-7fafaefe8b9e
+    # releases:
+    #  - name: cf-release
+    #    version: 132
+    # networks: {}
+    # properties:
+    #   cf:
+    #     dns: mycloud.com
+    #     ip_addresses: ['1.2.3.4']
+    #     deployment_size: medium
+    #     security_group: cf
+    #     persistent_disk: 4096
+    #
+    # It is then merged with the corresponding template from +release_version_cpi_size+.
+    def create_deployment_file
+      step("Creating deployment file #{deployment_file}",
+           "Failed to create deployment file #{deployment_file}", :fatal) do
+        File.open(deployment_file, "w") do |file|
+          file << {
+            "name" => deployment_attributes.name,
+            "director_uuid" => bosh_uuid,
+            "releases" => {
+              "name" => release_name,
+              "version" => release_version_number
+            },
+            "networks" => {},
+            "properties" => {
+              self.class.properties_key => deployment_attributes.attributes_with_string_keys
+            }
+          }.to_yaml
+        end
+
+        quieten_output do
+          deployment_cmd(non_interactive: true).set_current(deployment_file)
+          biff_cmd(non_interactive: true).biff(template_file)
+        end
+      end
+    rescue Bosh::Cli::ValidationHalted
+      errors.each do |error|
+        say error.make_red
+      end
+    end
+
+    # Perform the create/update deployment described by this +DeploymentFile+
+    def deploy(options={})
+      # set current deployment to show the change in the output
+      deployment_cmd.set_current(deployment_file)
+      deployment_cmd(non_interactive: options[:non_interactive]).perform
+    end
+
+    # The trio of DeploymentFile, DeploymentAttributes & ReleaseVersionCpiSize can be
+    # reconstructed from a deployment file that was previously generated by this class.
+    #
+    # Specifically, it requires the deployment file to look like:
+    # ---
+    # name: NAME
+    # releases:
+    #  - name: cf-release
+    #    version: 132
+    # properties:
+    #   cf:
+    #     dns: mycloud.com
+    #     ip_addresses: ['1.2.3.4']
+    #     deployment_size: medium
+    #     security_group: cf
+    #     persistent_disk: 4096
+    def self.reconstruct_from_deployment_file(deployment_file_path, director_client, bosh_status)
+      deployment_file = YAML.load_file(deployment_file_path)
+      bosh_cpi = bosh_status["cpi"]
+
+      release = deployment_file["releases"].find do |release|
+        release["name"] == "cf" || release["name"] == "cf-release"
+      end
+      release_version = release["version"]
+      release_version_cpi = ReleaseVersionCpi.new(release_version, bosh_cpi)
+      deployment_size = deployment_file["properties"]["deployment_size"]
+      release_version_cpi_size = ReleaseVersionCpiSize.new(release_version_cpi, deployment_size)
+
+      attributes = deployment_file["properties"][properties_key]
+      # convert string keys to symbol keys
+      attributes = attributes.inject({}) do |mem, key_value|
+        k, v = key_value; mem[k.to_sym] = v; mem
+      end
+      deployment_attributes = DeploymentAttributes.new(director_client, bosh_status, release_version_cpi, attributes)
+
+      self.new(release_version_cpi_size, deployment_attributes, bosh_status)
+    end
+
+    def release_name
+      release_version_cpi_size.release_name
+    end
+
+    def release_version_number
+      release_version_cpi_size.release_version_number
+    end
+
+    def deployment_size
+      deployment_attributes.deployment_size
+    end
+
+    def deployment_file
+      File.join(deployment_file_dir, "#{deployment_attributes.name}.yml")
+    end
+
+    def deployment_file_dir
+      File.expand_path("deployments/cf")
+    end
+
+    def template_file
+      release_version_cpi_size.template_file_path
+    end
+
+    def deployment_cmd(options = {})
+      cmd ||= Bosh::Cli::Command::Deployment.new
+      options.each do |key, value|
+        cmd.add_option key.to_sym, value
+      end
+      cmd
+    end
+
+    def release_cmd(options = {})
+      cmd ||= Bosh::Cli::Command::Release.new
+      options.each do |key, value|
+        cmd.add_option key.to_sym, value
+      end
+      cmd
+    end
+
+    def biff
+      @biff_cmd ||= Bosh::Cli::Command::Biff.new
+    end
+
+    def biff_cmd(options = {})
+      options.each do |key, value|
+        biff.add_option key.to_sym, value
+      end
+      biff
+    end
+
+    def bosh_uuid
+      bosh_status["uuid"]
+    end
+
+    def bosh_cpi
+      bosh_status["cpi"]
+    end
+
+    # attributes are stored within deployment file at properties.cf
+    def self.properties_key
+      "cf"
+    end
+
+    def quieten_output(&block)
+      stdout = Bosh::Cli::Config.output
+      Bosh::Cli::Config.output = nil
+      yield
+      Bosh::Cli::Config.output = stdout
+    end
+  end
+end