bosh_openstack_cpi 0.0.5 → 0.0.6

data/Rakefile

@@ -14,6 +14,16 @@ begin
 rescue LoadError
 end
 
+begin
+  require "bundler_task"
+rescue LoadError
+end
+
+begin
+  require "ci_task"
+rescue LoadError
+end
+
 gem_helper = Bundler::GemHelper.new(Dir.pwd)
 
 desc "Build CPI gem into the pkg directory"
@@ -21,10 +31,14 @@ task "build" do
   gem_helper.build_gem
 end
 
-desc "Build and install CPI into system gems"
-task "install" do
-  Rake::Task["bundler:install"].invoke
-  gem_helper.install_gem
+if defined?(BundlerTask)
+  BundlerTask.new
+
+  desc "Build and install CPI into system gems"
+  task "install" do
+    Rake::Task["bundler:install"].invoke
+    gem_helper.install_gem
+  end
 end
 
 if defined?(RSpec)
@@ -34,6 +48,12 @@ if defined?(RSpec)
       t.pattern = "spec/unit/**/*_spec.rb"
       t.rspec_opts = %w(--format progress --colour)
     end
+
+    if defined?(CiTask)
+      CiTask.new do |task|
+        task.rspec_task = rspec_task
+      end
+    end
   end
 
   desc "Run tests"

data/USAGE.md

@@ -0,0 +1,103 @@
+# BOSH OpenStack Cloud Provider Interface
+# Copyright (c) 2012 Piston Cloud Computing, Inc.
+
+For online documentation see: http://rubydoc.info/gems/bosh_openstack_cpi/
+
+## Options
+
+These options are passed to the OpenStack CPI when it is instantiated.
+
+### OpenStack options
+
+* `auth_url` (required)
+  URL of the OpenStack Identity endpoint to connect to
+* `username` (required)
+  OpenStack user name
+* `api_key` (required)
+  OpenStack API key
+* `tenant` (required)
+  OpenStack tenant name
+* `region` (optional)
+  OpenStack region
+* `default_key_name` (required)
+  default OpenStack ssh key name to assign to created virtual machines
+* `default_security_group` (required)
+  default OpenStack security group to assign to created virtual machines
+* `private_key` (required)
+  local path to the ssh private key, must match `default_key_name`
+
+### Registry options
+
+The registry options are passed to the Openstack CPI by the BOSH director based on the settings in `director.yml`, but can be overridden if needed.
+
+* `endpoint` (required)
+  OpenStack registry URL
+* `user` (required)
+  OpenStack registry user
+* `password` (required)
+  rOpenStack egistry password
+
+### Agent options
+
+Agent options are passed to the OpenStack  CPI by the BOSH director based on the settings in `director.yml`, but can be overridden if needed.
+
+### Resource pool options
+
+These options are specified under `cloud_options` in the `resource_pools` section of a BOSH deployment manifest.
+
+* `instance_type` (required)
+  which type of instance (OpenStack flavor) the VMs should belong to
+* `availability_zone` (optional)
+  the OpenStack availability zone the VMs should be created in
+
+### Network options
+
+These options are specified under `cloud_options` in the `networks` section of a BOSH deployment manifest.
+
+* `type` (required)
+  can be either `dynamic` for a DHCP assigned IP by OpenStack, or `vip` to use a Floating IP (which needs to be already allocated)
+
+## Example
+
+This is a sample of how OpenStack specific properties are used in a BOSH deployment manifest:
+
+    ---
+    name: sample
+    director_uuid: 38ce80c3-e9e9-4aac-ba61-97c676631b91
+
+    ...
+
+    networks:
+      - name: nginx_network
+        type: vip
+        cloud_properties: {}
+      - name: default
+        type: dynamic
+        cloud_properties:
+          security_groups:
+          - default
+
+    ...
+
+    resource_pools:
+      - name: common
+        network: default
+        size: 3
+        stemcell:
+          name: bosh-stemcell
+          version: 0.6.7
+        cloud_properties:
+          instance_type: m1.small
+
+    ...
+
+    properties:
+      openstack:
+        auth_url: http://pistoncloud.com/:5000/v2.0/tokens
+        username: christopher
+        api_key: QRoqsenPsNGX6
+        tenant: Bosh
+        region: us-west
+        default_key_name: bosh
+        default_security_groups: ["bosh"]
+        private_key: /home/bosh/.ssh/bosh.pem

data/bosh_cpi.md

@@ -0,0 +1,82 @@
+#OpenStack CPI Implementation #
+OpenStack CPI is an implementation of BOSH CPI. It allows BOSH to interface with various services in OpenStack like glance, controller and registry. In the sections below we outline the implementation details for each of the method in CPI interface.
+
+##Initialize ##
+
+Implementation of `def initialize(options)` method. 
+
+1. Validate the `options` passed to this method
+2. In the second step the parameters are extracted from `options` object to populate the following properties 
+	+ `@agent_properties`	
+	+ `@openstack_properties`
+	+ `@registry_propertiess`
+3. Populate the `openstack_params` and connect to remove Nova Service
+	1. Instantiate `Fog::Compute` instance
+	2. `Fog::Compute::OpenStack` instance is created
+	3. A New `Fog::Connection` object connects with the remove Nova Compute Service
+4. Populate the `glance_params`and connect to remove Glance service
+       1.  Instantiate `Fog::Image` instance
+       2.  `Fog::Image::OpenStack` instance is created
+       3.  A New `Fog::Connection` object connects with the remove Glance Service
+5. Instantiate `Registry_Client`.
+
+Figure below shows the flow of control.
+
+![openstack_cpi_initialize](https://raw.github.com/piston/openstack-bosh-cpi/master/images/openstack_cpi_initialize.png)
+
+##Create Stemcell ##
+
+Implementation of method `create_stemcell(image_path, cloud_properties)`
+Steps outlined below are the flow control implemented to extract and upload kernel image, ramdisk and stem cell image.
+
+1. Extract parameters from the `cloud_properties`. Check if `kernel_file` parameter exists, instantiate `kernel_image` object
+2. Construct `kernel_params`
+3. Upload `kernel_image` to glance service by calling the method `upload_image(kernel_params)`
+4. If params contain `ramdisk_file` 
+5. Instantiate ramdisk_image object and populate `ramdisk_parama`
+6. Upload the `ramdisk_image` to glance service by calling `upload(ramdisk_params)`
+7. Populate `image_params` for the stem cell to be uploaded to glance service
+8. Call the method `upload_image(image_params)` 
+
+Figure below shows the flow control for the method `create_stemcell(image_path, cloud_properties)`
+
+![openstack_cpi_createstemcell](https://raw.github.com/piston/openstack-bosh-cpi/master/images/openstack_cpi_createstemcell.png)
+
+##Delete Stemcell
+
+![openstack_cpi_deletestemcell](https://raw.github.com/piston/openstack-bosh-cpi/master/images/openstack_cpi_deletestemcell.png)
+
+##Create VM ##
+
+![openstack_cpi_create_vm](https://raw.github.com/piston/openstack-bosh-cpi/master/images/openstack_cpi_create_vm.png)
+
+##Delete VM ##
+
+Implementation of `delete_vm(server_id)`. This method deletes the VM created in Nova Compute.
+
+1. Get the `server_id` of the VM to be deleted
+	* 1.1, 1.2 : Send the request `get_server_details` to Compute API Server through  `Fog::Connection`
+2.  If `server` object returned is not null call `server.destroy`. This will send `delete_server` request to Nova Compute.
+	* 2.1, 2.2 : Create and send the `delete_server` request through `Fog::Connection` 
+3.  Delete the settings from Registry by calling `delete_settings` method.
+
+Figure below shows the flow control for `delete_vm` method
+
+![openstack_cpi_delete_vm](https://raw.github.com/piston/openstack-bosh-cpi/master/images/openstack_cpi_delete_vm.png)
+
+##Create Disk ##
+
+1. Check if size passed is integer, is greater than 1024 and less than 1024*1000, else throw an error
+2. Create `volume_params`
+3. Call `create()volume_params)` on `Fog::Compute` service
+     1. `create_volume` request on `Fog::Volume::OpenStack` 
+     2. Opens a `Fog::Connection` request to access the remote service and create a volume.
+
+Figure below shows the flow control of `create_disk` method
+
+![openstack_cpi_create_disk](https://raw.github.com/piston/openstack-bosh-cpi/master/images/openstack_cpi_create_disk.png)
+
+##Delete Disk##
+
+This method deletes the volume created in OpenStack Nova Volume 
+![openstack_cpi_delete_disk](https://raw.github.com/piston/openstack-bosh-cpi/master/images/openstack_cpi_delete_disk.png)

data/lib/cloud/openstack/cloud.rb

@@ -91,7 +91,7 @@ module Bosh::OpenStackCloud
             # 1. Unpack image to temp directory
             unpack_image(tmp_dir, image_path)
             root_image = File.join(tmp_dir, "root.img")
-
+        
             # 2. If image contains a kernel file, upload it to glance service
             kernel_id = nil
             if cloud_properties["kernel_id"]
@@ -122,7 +122,7 @@ module Bosh::OpenStackCloud
               ramdisk_id = cloud_properties["ramdisk_id"]
             elsif cloud_properties["ramdisk_file"]
               ramdisk_image = File.join(tmp_dir, cloud_properties["ramdisk_file"])
-              unless File.exists?(kernel_image)
+              unless File.exists?(ramdisk_image)
                 cloud_error("Ramdisk image " \
                             "#{cloud_properties['ramdisk_file']} " \
                             "is missing from stemcell archive")
@@ -234,18 +234,9 @@ module Bosh::OpenStackCloud
                   network_spec = nil, disk_locality = nil, environment = nil)
       with_thread_name("create_vm(#{agent_id}, ...)") do
         @logger.info("Creating new server...")
-        network_configurator = NetworkConfigurator.new(network_spec)
-
         server_name = "vm-#{generate_unique_name}"
-        user_data = {
-          "registry" => {
-            "endpoint" => @registry.endpoint
-          },
-          "server" => {
-            "name" => server_name
-          }
-        }
 
+        network_configurator = NetworkConfigurator.new(network_spec)
         security_groups =
           network_configurator.security_groups(@default_security_groups)
         @logger.debug("Using security groups: `#{security_groups.join(', ')}'")
@@ -269,7 +260,8 @@ module Bosh::OpenStackCloud
           :flavor_ref => flavor.id,
           :key_name => resource_pool["key_name"] || @default_key_name,
           :security_groups => security_groups,
-          :user_data => Yajl::Encoder.encode(user_data)
+          :user_data => Yajl::Encoder.encode(user_data(server_name,
+                                                       network_spec))
         }
 
         availability_zone = select_availability_zone(disk_locality,
@@ -489,6 +481,26 @@ module Bosh::OpenStackCloud
       end
     end
 
+    ##
+    # Set metadata for an OpenStack server
+    #
+    # @param [String] server_id OpenStack server UUID
+    # @param [Hash] metadata Metadata key/value pairs
+    # @return [void]
+    def set_vm_metadata(server_id, metadata)
+      with_thread_name("set_vm_metadata(#{server_id}, ...)") do
+        server = @openstack.servers.get(server_id)
+        unless server
+          cloud_error("Server `#{server_id}' not found")
+        end
+
+        metadata.each do |name, value|
+          value = "" if value.nil? # value is required
+          server.metadata.update(name => value)
+        end
+      end
+    end
+
     ##
     # Validates the deployment
     #
@@ -543,6 +555,39 @@ module Bosh::OpenStackCloud
       UUIDTools::UUID.random_create.to_s
     end
 
+    ##
+    # Prepare server user data
+    #
+    # @param [String] server_name server name
+    # @param [Hash] network_spec network specification
+    # @return [Hash] server user data
+    def user_data(server_name, network_spec)
+      data = {}
+
+      data["registry"] = { "endpoint" => @registry.endpoint }
+      data["server"] = { "name" => server_name }
+
+      with_dns(network_spec) do |servers|
+        data["dns"] = { "nameserver" => servers }
+      end
+
+      data
+    end
+
+    ##
+    # Extract dns server list from network spec and yield the the list
+    #
+    # @param [Hash] network_spec network specification for instance
+    # @yield [Array]
+    def with_dns(network_spec)
+      network_spec.each_value do |properties|
+        if properties.has_key?("dns") && !properties["dns"].nil?
+          yield properties["dns"]
+          return
+        end
+      end
+    end
+
     ##
     # Generates initial agent settings. These settings will be read by agent
     # from OpenStack registry (also a BOSH component) on a target server. Disk
@@ -696,7 +741,6 @@ module Bosh::OpenStackCloud
         cloud_error("Failed to unpack stemcell root image" \
                     "tar exit status #{$?.exitstatus}: #{output}")
       end
-
       root_image = File.join(tmp_dir, "root.img")
       unless File.exists?(root_image)
         cloud_error("Root image is missing from stemcell archive")
@@ -710,8 +754,8 @@ module Bosh::OpenStackCloud
     # @return [void]
     # @raise [ArgumentError] if options are not valid
     def validate_options
-      unless @options.has_key?("openstack") &&
-          @options["openstack"].is_a?(Hash) &&
+      unless @options["openstack"].is_a?(Hash) &&
+          @options.has_key?("openstack") &&
           @options["openstack"]["auth_url"] &&
           @options["openstack"]["username"] &&
           @options["openstack"]["api_key"] &&

data/lib/cloud/openstack/version.rb

@@ -2,6 +2,6 @@
 
 module Bosh
   module OpenStackCloud
-    VERSION = "0.0.5"
+    VERSION = "0.0.6"
   end
 end

data/spec/spec_helper.rb

@@ -9,6 +9,9 @@ Bundler.setup(:default, :test)
 
 require "rspec"
 require "tmpdir"
+require 'zlib'
+require 'archive/tar/minitar'
+include Archive::Tar
 
 require "cloud/openstack"
 

data/spec/unit/cloud_spec.rb

@@ -13,6 +13,40 @@ describe Bosh::OpenStackCloud::Cloud do
       cloud.should be_an_instance_of(Bosh::OpenStackCloud::Cloud)
     end
 
-  end
+    it "raises ArgumentError on initializing with blank options" do
+    	options = Hash.new("options")
+    	expect { 
+    		Bosh::OpenStackCloud::Cloud.new(options)
+    	}.to raise_error(ArgumentError)
+    end
+
+    it "raises ArgumentError on initializing with non Hashoptions" do
+    	options = "this is a string"
+    	expect { 
+    		Bosh::OpenStackCloud::Cloud.new(options)
+    	}.to raise_error(ArgumentError)
+    end
 
+    it "should create a Cloud Instance on giving valid params" do
+      openstack = { "auth_url" => "http://localhost/",
+                    "username" =>"testuser",
+          				  "api_key" => "test_api_key",
+          				 "tenant" => "test_tenant"
+          			  }
+      registry = { "endpoint" => "http://0.0.0.0",
+          	       "user" => "testuser",
+          	       "password" => "password"
+                 }
+      options = { "openstack" => openstack,
+                  "registry" => registry
+                }
+      socket_error = false
+      begin
+        cloud = Bosh::OpenStackCloud::Cloud.new(options)
+        rescue Excon::Errors::SocketError => e
+          socket_error = true
+        end
+      socket_error.should be_true
+    end
+  end
 end

data/spec/unit/create_stemcell_spec.rb

@@ -196,6 +196,120 @@ describe Bosh::OpenStackCloud::Cloud do
       sc_id.should == "i-bar"
     end
 
-  end
+    it "should throw an error for non existent root image in stemcell archive" do
+      begin 
+        dir = Dir.mkdir("tmp")
+        Dir.chdir 'tmp'    
+   
+        kernel_img = File.new('kernel.img', 'w')
+        ramdisk_img = File.new('initrd.img', 'w')
+
+        tgz = Zlib::GzipWriter.new(File.open('stemcell.tgz', 'wb'))
+        array = ['kernel.img']
+    
+        Minitar.pack(array, tgz)
+      
+        cloud = mock_glance
+        error_expected = "Root image is missing from stemcell archive"
+        error_actual = ""
+        begin
+          cloud.create_stemcell("./stemcell.tgz", {
+            "name" => "bosh-stemcell",
+            "version" => "x.y.z",
+            "container_format" => "ami",
+            "disk_format" => "ami",
+            "kernel_file" => "kernel.img",
+            "ramdisk_file" => "initrd.img"
+          })
+        rescue Bosh::Clouds::CloudError => e
+          error_actual  = e.to_s
+        end
+        error_actual.should eq(error_expected)
+      ensure
+        File.delete('kernel.img')
+        File.delete('initrd.img')
+        File.delete('stemcell.tgz')
+        Dir.chdir '..'
+        Dir.delete('tmp')
+      end
+    end
+
+    it "should throw an error for non existent kernel image in stemcell archive" do
+      begin 
+        dir = Dir.mkdir("tmp")
+        Dir.chdir 'tmp'    
+        root_img = File.new('root.img','w') 
+        if root_img
+          root_img.syswrite("ABCDEF")
+        end
+        ramdisk_img = File.new('initrd.img', 'w')
+
+        tgz = Zlib::GzipWriter.new(File.open('stemcell.tgz', 'wb'))
+        array = ['root.img']
+    
+        Minitar.pack(array, tgz)
+      
+        cloud = mock_glance
+        error_expected = "Kernel image kernel.img is missing from stemcell archive"
+        error_actual = ""
+        begin
+          cloud.create_stemcell("./stemcell.tgz", {
+            "name" => "bosh-stemcell",
+            "version" => "x.y.z",
+            "container_format" => "ami",
+            "disk_format" => "ami",
+            "kernel_file" => "kernel.img",
+            "ramdisk_file" => "initrd.img"
+          })
+        rescue Bosh::Clouds::CloudError => e
+          error_actual  = e.to_s
+        end
+        error_actual.should eq(error_expected)
+      ensure
+        File.delete('root.img')
+        File.delete('initrd.img')
+        File.delete('stemcell.tgz')
+        Dir.chdir '..'
+        Dir.delete('tmp')
+      end
+    end
+
+    it "should throw an error for non existent ramdisk image in stemcell archive" do
+      begin 
+        dir = Dir.mkdir("tmp")
+        Dir.chdir 'tmp'    
+        root_img = File.new('root.img','w') 
+        if root_img
+          root_img.syswrite("ABCDEF")
+        end
 
+        tgz = Zlib::GzipWriter.new(File.open('stemcell.tgz', 'wb'))
+        array = ['root.img']
+    
+        Minitar.pack(array, tgz)
+      
+        cloud = mock_glance
+        error_expected = "Ramdisk image initrd.img is missing from stemcell archive"
+        error_actual = ""
+        begin
+          cloud.create_stemcell("./stemcell.tgz", {
+            "name" => "bosh-stemcell",
+            "version" => "x.y.z",
+            "container_format" => "ami",
+            "disk_format" => "ami",
+            #"kernel_file" => "kernel.img",
+            "ramdisk_file" => "initrd.img"
+          })
+        rescue Bosh::Clouds::CloudError => e
+          error_actual  = e.to_s
+        end
+        error_actual.should eq(error_expected)
+      ensure
+        File.delete('root.img')
+        File.delete('stemcell.tgz')
+        Dir.chdir '..'
+        Dir.delete('tmp')
+      end
+    end
+  end
 end

data/spec/unit/create_vm_spec.rb

@@ -79,6 +79,52 @@ describe Bosh::OpenStackCloud::Cloud, "create_vm" do
     vm_id.should == "i-test"
   end
 
+  it "passes dns servers in server user data when present" do
+    unique_name = UUIDTools::UUID.random_create.to_s
+
+    user_data = {
+        "registry" => {
+          "endpoint" => "http://registry:3333"
+        },
+        "server" => {
+          "name" => "vm-#{unique_name}"
+        },
+        "dns" => {
+          "nameserver" => ["1.2.3.4"]
+        }
+    }
+    server = double("server", :id => "i-test", :name => "i-test")
+    image = double("image", :id => "sc-id", :name => "sc-id")
+    flavor = double("flavor", :id => "f-test", :name => "m1.tiny")
+    address = double("address", :id => "a-test", :ip => "10.0.0.1",
+                     :instance_id => "i-test")
+    network_spec = dynamic_network_spec
+    network_spec["dns"] = ["1.2.3.4"]
+
+    cloud = mock_cloud do |openstack|
+      openstack.servers.should_receive(:create).
+          with(openstack_params(unique_name, user_data, %w[default])).
+          and_return(server)
+      openstack.images.should_receive(:find).and_return(image)
+      openstack.flavors.should_receive(:find).and_return(flavor)
+      openstack.addresses.should_receive(:each).and_yield(address)
+    end
+
+    cloud.should_receive(:generate_unique_name).and_return(unique_name)
+    address.should_receive(:server=).with(nil)
+    cloud.should_receive(:wait_resource).with(server, :active, :state)
+
+    @registry.should_receive(:update_settings).
+        with("i-test", agent_settings(unique_name, network_spec))
+
+    vm_id = cloud.create_vm("agent-id", "sc-id",
+                            resource_pool_spec,
+                            { "network_a" => network_spec },
+                            nil, { "test_env" => "value" })
+    vm_id.should == "i-test"
+  end
+
+
   it "creates an OpenStack server with security group" do
     unique_name = UUIDTools::UUID.random_create.to_s
     user_data = {

data/spec/unit/set_vm_metadata_spec.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require File.expand_path("../../spec_helper", __FILE__)
+
+describe Bosh::OpenStackCloud::Cloud do
+  before :each do
+    @server = double("server", :id => "i-foobar")
+    @metadata = double("metadata")
+
+    @cloud = mock_cloud do |openstack|
+      openstack.servers.should_receive(:get).
+        with("i-foobar").and_return(@server)
+    end
+  end
+
+  it "should set metadata" do
+    metadata = {:job => "job", :index => "index"}
+
+    @server.should_receive(:metadata).and_return(@metadata, @metadata)
+    @metadata.should_receive(:update).with(:job => "job")
+    @metadata.should_receive(:update).with(:index => "index")
+
+    @cloud.set_vm_metadata("i-foobar", metadata)
+  end
+
+  it "should set metadata with a nil value" do
+    metadata = {:deployment => nil}
+
+    @server.should_receive(:metadata).and_return(@metadata)
+    @metadata.should_receive(:update).with(:deployment => "")
+
+    @cloud.set_vm_metadata("i-foobar", metadata)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh_openstack_cpi
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-10 00:00:00.000000000 Z
+date: 2013-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -126,6 +126,8 @@ files:
 - lib/cloud/openstack/version.rb
 - lib/cloud/openstack/vip_network.rb
 - README.md
+- USAGE.md
+- bosh_cpi.md
 - Rakefile
 - spec/assets/sample_config.yml
 - spec/integration/cpi_test.rb
@@ -143,6 +145,7 @@ files:
 - spec/unit/helpers_spec.rb
 - spec/unit/network_configurator_spec.rb
 - spec/unit/reboot_vm_spec.rb
+- spec/unit/set_vm_metadata_spec.rb
 - spec/unit/validate_deployment_spec.rb
 homepage: http://www.pistoncloud.com
 licenses: []
@@ -185,5 +188,6 @@ test_files:
 - spec/unit/helpers_spec.rb
 - spec/unit/network_configurator_spec.rb
 - spec/unit/reboot_vm_spec.rb
+- spec/unit/set_vm_metadata_spec.rb
 - spec/unit/validate_deployment_spec.rb
 has_rdoc: