bosh_openstack_cpi 0.0.4 → 0.0.5

data/lib/cloud/openstack/dynamic_network.rb

@@ -2,7 +2,7 @@
 
 module Bosh::OpenStackCloud
   ##
-  #
+  # Represents OpenStack dynamic network: where IaaS sets VM's IP
   class DynamicNetwork < Network
 
     ##
@@ -17,8 +17,10 @@ module Bosh::OpenStackCloud
     ##
     # Configures OpenStack dynamic network. Right now it's a no-op,
     # as dynamic networks are completely managed by OpenStack
+    #
     # @param [Fog::Compute::OpenStack] openstack Fog OpenStack Compute client
-    # @param [Fog::Compute::OpenStack::Server] server OpenStack server to configure
+    # @param [Fog::Compute::OpenStack::Server] server OpenStack server to
+    #   configure
     def configure(openstack, server)
     end
 

data/lib/cloud/openstack/helpers.rb

@@ -4,44 +4,74 @@ module Bosh::OpenStackCloud
 
   module Helpers
 
-    DEFAULT_TIMEOUT = 3600
+    DEFAULT_TIMEOUT = 600 # Default timeout for target state (in seconds)
 
-    #
+    ##
     # Raises CloudError exception
     #
+    # @param [String] message Message about what went wrong
     def cloud_error(message)
       @logger.error(message) if @logger
       raise Bosh::Clouds::CloudError, message
     end
 
-    #
+    ##
     # Waits for a resource to be on a target state
     #
-    def wait_resource(resource, start_state, target_state, state_method = :status, timeout = DEFAULT_TIMEOUT)
+    # @param [Fog::Model] resource Resource to query
+    # @param [Symbol] target_state Resource's state desired
+    # @param [Symbol] state_method Resource's method to fetch state
+    # @param [Boolean] allow_notfound true if resource could be not found
+    # @param [Integer] timeout Timeout for target state (in seconds)
+    def wait_resource(resource, target_state, state_method = :status,
+                      allow_notfound = false, timeout = DEFAULT_TIMEOUT)
 
       started_at = Time.now
-      state = resource.send(state_method).downcase rescue state_method
-      desc = resource.class.name.split("::").last.to_s + " " + resource.id.to_s
+      desc = resource.class.name.split("::").last.to_s + " `" +
+             resource.id.to_s + "'"
+
+      loop do
+        task_checkpoint
 
-      while state.to_sym != target_state
         duration = Time.now - started_at
 
         if duration > timeout
           cloud_error("Timed out waiting for #{desc} to be #{target_state}")
         end
 
-        @logger.debug("Waiting for #{desc} to be #{target_state} (#{duration})") if @logger
-
-        sleep(1)
+        if @logger
+          @logger.debug("Waiting for #{desc} to be #{target_state} " \
+                        "(#{duration}s)")
+        end
 
+        # If resource reload is nil, perhaps it's because resource went away
+        # (ie: a destroy operation). Don't raise an exception if this is
+        # expected (allow_notfound).
         if resource.reload.nil?
-          state = target_state
+          break if allow_notfound
+          cloud_error("#{desc}: Resource not found")
         else
-          state = resource.send(state_method).downcase rescue state_method
+          state = resource.send(state_method).downcase.to_sym
         end
+
+        # This is not a very strong convention, but some resources
+        # have 'error' and 'failed' states, we probably don't want to keep
+        # waiting if we're in these states. Alternatively we could introduce a
+        # set of 'loop breaker' states but that doesn't seem very helpful
+        # at the moment
+        if state == :error || state == :failed
+          cloud_error("#{desc} state is #{state}, expected #{target_state}")
+        end
+
+        break if state == target_state
+
+        sleep(1)
       end
 
-      @logger.info("#{desc} is #{target_state} after #{Time.now - started_at}s") if @logger
+      if @logger
+        total = Time.now - started_at
+        @logger.info("#{desc} is now #{target_state}, took #{total}s")
+      end
     end
 
   end

data/lib/cloud/openstack/network.rb

@@ -2,7 +2,7 @@
 
 module Bosh::OpenStackCloud
   ##
-  #
+  # Represents OpenStack network.
   class Network
     include Helpers
 

data/lib/cloud/openstack/network_configurator.rb

@@ -8,15 +8,13 @@ module Bosh::OpenStackCloud
   # a number of sanity checks for the network spec provided by director
   # to make sure we don't apply something OpenStack doesn't understand how to
   # deal with.
-  #
   class NetworkConfigurator
     include Helpers
 
     ##
     # Creates new network spec
     #
-    # @param [Hash] spec raw network spec passed by director
-    # TODO Add network configuration examples
+    # @param [Hash] spec Raw network spec passed by director
     def initialize(spec)
       unless spec.is_a?(Hash)
         raise ArgumentError, "Invalid spec, Hash expected, " \
@@ -37,20 +35,19 @@ module Bosh::OpenStackCloud
             cloud_error("More than one dynamic network for `#{name}'")
           else
             @dynamic_network = DynamicNetwork.new(name, spec)
-            # only extract security groups for dynamic networks
-            extract_security_groups(spec)
+            @security_groups += extract_security_groups(spec)
           end
         when "vip"
           if @vip_network
             cloud_error("More than one vip network for `#{name}'")
           else
             @vip_network = VipNetwork.new(name, spec)
+            @security_groups += extract_security_groups(spec)
           end
         else
-          cloud_error("Invalid network type `#{network_type}': OpenStack CPI " \
-                      "can only handle `dynamic' and `vip' network types")
+          cloud_error("Invalid network type `#{network_type}': OpenStack " \
+                      "CPI can only handle `dynamic' and `vip' network types")
         end
-
       end
 
       if @dynamic_network.nil?
@@ -58,6 +55,12 @@ module Bosh::OpenStackCloud
       end
     end
 
+    ##
+    # Applies network configuration to the vm
+    #
+    # @param [Fog::Compute::OpenStack] openstack Fog OpenStack Compute client
+    # @param [Fog::Compute::OpenStack::Server] server OpenStack server to
+    #   configure
     def configure(openstack, server)
       @dynamic_network.configure(openstack, server)
 
@@ -81,33 +84,38 @@ module Bosh::OpenStackCloud
     # Returns the security groups for this network configuration, or
     # the default security groups if the configuration does not contain
     # security groups
+    #
     # @param [Array] default Default security groups
     # @return [Array] security groups
     def security_groups(default)
       if @security_groups.empty? && default
-        return default
+        default
       else
-        return @security_groups
+        @security_groups.sort
       end
     end
 
+    private
+
     ##
     # Extracts the security groups from the network configuration
+    #
     # @param [Hash] network_spec Network specification
+    # @return [Array] security groups
     # @raise [ArgumentError] if the security groups in the network_spec
     #   is not an Array
-    def extract_security_groups(spec)
-      if spec && spec["cloud_properties"]
-        cloud_properties = spec["cloud_properties"]
+    def extract_security_groups(network_spec)
+      if network_spec && network_spec["cloud_properties"]
+        cloud_properties = network_spec["cloud_properties"]
         if cloud_properties && cloud_properties["security_groups"]
           unless cloud_properties["security_groups"].is_a?(Array)
             raise ArgumentError, "security groups must be an Array"
           end
-          @security_groups += cloud_properties["security_groups"]
+          return cloud_properties["security_groups"]
         end
       end
+      []
     end
 
   end
-
 end

data/lib/cloud/openstack/registry_client.rb

@@ -1,6 +1,23 @@
 # Copyright (c) 2012 Piston Cloud Computing, Inc.
 
 module Bosh::OpenStackCloud
+  ##
+  # Represents OpenStack Registry Client. It performs CRUD operations against
+  # the OpenStack Registry.
+  #
+  # Settings example:
+  # settings = {
+  #   "vm" => {
+  #     "name" => server_name
+  #   },
+  #   "agent_id" => agent_id,
+  #   "networks" => network_spec,
+  #   "disks" => {
+  #     "system" => "/dev/vda",
+  #     "ephemeral" => "/dev/vdb",
+  #     "persistent" => {"volume_id" => device_name}
+  #   }
+  # }
   class RegistryClient
     include Helpers
 
@@ -8,6 +25,12 @@ module Bosh::OpenStackCloud
     attr_reader :user
     attr_reader :password
 
+    ##
+    # Creates a new Registry client
+    #
+    # @param [String] endpoint Registry endpoint URL
+    # @param [String] user Registry user
+    # @param [String] password Registry password
     def initialize(endpoint, user, password)
       @endpoint = endpoint
 
@@ -17,7 +40,6 @@ module Bosh::OpenStackCloud
 
       @user = user
       @password = password
-
       auth = Base64.encode64("#{@user}:#{@password}").gsub("\n", "")
 
       @headers = {
@@ -30,6 +52,7 @@ module Bosh::OpenStackCloud
 
     ##
     # Update server settings in the registry
+    #
     # @param [String] server_id OpenStack server id
     # @param [Hash] settings New agent settings
     # @return [Boolean]
@@ -42,8 +65,7 @@ module Bosh::OpenStackCloud
       payload = Yajl::Encoder.encode(settings)
       url = "#{@endpoint}/servers/#{server_id}/settings"
 
-      response = @client.put(url, payload, @headers)
-
+      response = @client.put(url, {:body => payload, :header => @headers})
       if response.status != 200
         cloud_error("Cannot update settings for `#{server_id}', " \
                     "got HTTP #{response.status}")
@@ -54,27 +76,25 @@ module Bosh::OpenStackCloud
 
     ##
     # Read server settings from the registry
+    #
     # @param [String] server_id OpenStack server id
     # @return [Hash] Agent settings
     def read_settings(server_id)
       url = "#{@endpoint}/servers/#{server_id}/settings"
 
-      response = @client.get(url, {}, @headers)
-
+      response = @client.get(url, {:header => @headers})
       if response.status != 200
         cloud_error("Cannot read settings for `#{server_id}', " \
                     "got HTTP #{response.status}")
       end
 
       body = Yajl::Parser.parse(response.body)
-
       unless body.is_a?(Hash)
         cloud_error("Invalid registry response, Hash expected, " \
                     "got #{body.class}: #{body}")
       end
 
       settings = Yajl::Parser.parse(body["settings"])
-
       unless settings.is_a?(Hash)
         cloud_error("Invalid settings format, " \
                     "Hash expected, got #{settings.class}: " \
@@ -82,20 +102,19 @@ module Bosh::OpenStackCloud
       end
 
       settings
-
     rescue Yajl::ParseError
       cloud_error("Cannot parse settings for `#{server_id}'")
     end
 
     ##
     # Delete server settings from the registry
+    #
     # @param [String] server_id OpenStack server id
     # @return [Boolean]
     def delete_settings(server_id)
       url = "#{@endpoint}/servers/#{server_id}/settings"
 
-      response = @client.delete(url, @headers)
-
+      response = @client.delete(url, {:header => @headers})
       if response.status != 200
         cloud_error("Cannot delete settings for `#{server_id}', " \
                     "got HTTP #{response.status}")
@@ -105,5 +124,4 @@ module Bosh::OpenStackCloud
     end
 
   end
-
 end

data/lib/cloud/openstack/version.rb

@@ -2,6 +2,6 @@
 
 module Bosh
   module OpenStackCloud
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

data/lib/cloud/openstack/vip_network.rb

@@ -2,7 +2,8 @@
 
 module Bosh::OpenStackCloud
   ##
-  #
+  # Represents OpenStack vip network: where users sets VM's IP (floating IP's
+  # in OpenStack)
   class VipNetwork < Network
 
     ##
@@ -15,33 +16,31 @@ module Bosh::OpenStackCloud
     end
 
     ##
-    # Configures vip network
+    # Configures OpenStack vip network
     #
     # @param [Fog::Compute::OpenStack] openstack Fog OpenStack Compute client
-    # @param [Fog::Compute::OpenStack::Server] server OpenStack server to configure
+    # @param [Fog::Compute::OpenStack::Server] server OpenStack server to
+    #   configure
     def configure(openstack, server)
       if @ip.nil?
         cloud_error("No IP provided for vip network `#{@name}'")
       end
 
-      @logger.info("Associating server `#{server.id}' " \
-                   "with floating IP `#{@ip}'")
-
-      # Check if the OpenStack floating IP is allocated. If true, check
-      # if it is associated to any server, so we can disassociate it
-      # before associating it to the new server.
-      address_id = nil
-      addresses = openstack.addresses
-      addresses.each do |address|
-        if address.ip == @ip
-          address.server = nil unless address.instance_id.nil?
-          address.server = server
-          address_id = address.id
-          break
+      # Check if the OpenStack floating IP is allocated. If true, disassociate
+      # it from any server before associating it to the new server
+      address = openstack.addresses.find { |a| a.ip == @ip }
+      if address
+        unless address.instance_id.nil?
+          @logger.info("Disassociating floating IP `#{@ip}' " \
+                       "from server `#{address.instance_id}'")
+          address.server = nil
         end
-      end
-      if address_id.nil?
-        cloud_error("OpenStack CPI: floating IP #{@ip} not allocated")
+
+        @logger.info("Associating server `#{server.id}' " \
+                     "with floating IP `#{@ip}'")
+        address.server = server
+      else
+        cloud_error("Floating IP #{@ip} not allocated")
       end
     end
 

data/spec/assets/sample_config.yml

@@ -0,0 +1,14 @@
+---
+openstack:
+  auth_url: http://127.0.0.1:5000/v2.0/tokens
+  username: foo
+  api_key: bar
+  tenant: bosh
+  region:
+  default_key_name: default
+  default_security_groups: ["default"]
+
+registry:
+  endpoint: http://127.0.0.1:25695
+  user: admin
+  password: admin

data/spec/integration/cpi_test.rb

@@ -0,0 +1,119 @@
+# Copyright (c) 2012 Piston Cloud Computing, Inc.
+
+require File.expand_path("../../spec_helper", __FILE__)
+require "tempfile"
+
+##
+# BOSH OpenStack CPI Integration tests
+#
+# To run integration test:
+# 1. Prepare OpenStack CPI configuration file (a sample can be found at
+#    spec/assets/sample_config.yml);
+# 2. Set CPI_CONFIG_FILE env variable to point to OpenStack config file;
+# 3. Download a public OpenStack stemcell: 'bosh download public stemcell ...';
+# 4. Untar the OpenStack stemcell, you'll find and image file;
+# 5. Set STEMCELL_FILE env variable to point to OpenStack image file;
+# 6. Optional: Set FLOATING_IP env variable with an allocated OpenStack
+#    Floating IP if you want to test vip networks;
+# 7. Start OpenStack Registry manually (see bosh/openstack_registry);
+# 8. Run 'bundle exec rspec --color spec/integration/cpi_test.rb'.
+describe Bosh::OpenStackCloud::Cloud do
+
+  before(:each) do
+    unless ENV["CPI_CONFIG_FILE"]
+      raise "Please provide CPI_CONFIG_FILE environment variable"
+    end
+    unless ENV["STEMCELL_FILE"]
+      raise "Please provide STEMCELL_UUID environment variable"
+    end
+    @config = YAML.load_file(ENV["CPI_CONFIG_FILE"])
+    @logger = Logger.new(STDOUT)
+    Bosh::Clouds::Config.stub(:task_checkpoint)
+  end
+
+  let(:cpi) do
+    cpi = Bosh::OpenStackCloud::Cloud.new(@config)
+    cpi.logger = @logger
+
+    # As we inject the configuration file from the outside, we don't care
+    # about spinning up the registry ourselves. However we don't want to bother
+    # OpenStack at all if registry is not working, so just in case we perform
+    # a test health check against whatever has been provided.
+    cpi.registry.update_settings("foo", { "bar" => "baz" })
+    cpi.registry.read_settings("foo").should == { "bar" => "baz"}
+    cpi.registry.delete_settings("foo")
+
+    cpi
+  end
+
+  it "exercises a VM lifecycle" do
+    unique_name = UUIDTools::UUID.random_create.to_s
+    cpi.stub(:generate_unique_name).and_return(unique_name)
+
+    stemcell_id = cpi.create_stemcell(
+        ENV["STEMCELL_FILE"],
+        {
+          "name" => "bosh-stemcell",
+          "version" => "0.6.7",
+          "infrastructure" => "openstack",
+          "disk_format" => "qcow2",
+          "container_format" => "bare"
+         })
+
+    server_id = cpi.create_vm(
+      "agent-007", stemcell_id,
+      { "instance_type" => "m1.small" },
+      { "default" => { "type" => "dynamic" }},
+      [], { "key" => "value" })
+
+    server_id.should_not be_nil
+
+    settings = cpi.registry.read_settings("vm-#{unique_name}")
+    settings["vm"].should be_a(Hash)
+    settings["vm"]["name"].should == "vm-#{unique_name}"
+    settings["agent_id"].should == "agent-007"
+    settings["networks"].should == { "default" => { "type" => "dynamic" }}
+    settings["disks"].should == {
+      "system" => "/dev/vda",
+      "ephemeral" => "/dev/vdb",
+      "persistent" => {}
+    }
+    settings["env"].should == { "key" => "value" }
+
+    volume_id = cpi.create_disk(1024)
+    volume_id.should_not be_nil
+
+    cpi.attach_disk(server_id, volume_id)
+    settings = cpi.registry.read_settings("vm-#{unique_name}")
+    settings["disks"]["persistent"].should == { volume_id => "/dev/vdc" }
+
+    cpi.reboot_vm(server_id)
+
+    if ENV["FLOATING_IP"]
+      cpi.configure_networks(server_id,
+                             { "default" => { "type" => "dynamic" },
+                               "floating" => { "type" => "vip",
+                                               "ip" => ENV["FLOATING_IP"] }
+                             })
+      settings = cpi.registry.read_settings("vm-#{unique_name}")
+      settings["networks"].should == {
+        "default" => { "type" => "dynamic" },
+        "floating" => { "type" => "vip",
+                        "ip" => ENV["FLOATING_IP"] }
+      }
+    end
+
+    cpi.detach_disk(server_id, volume_id)
+    settings = cpi.registry.read_settings("vm-#{unique_name}")
+    settings["disks"]["persistent"].should == {}
+
+    cpi.delete_vm(server_id)
+    cpi.delete_disk(volume_id)
+    cpi.delete_stemcell(stemcell_id)
+
+    expect {
+      cpi.registry.read_settings("vm-#{unique_name}")
+    }.to raise_error(/HTTP 404/)
+  end
+
+end