wakame-vdc-dcmgr 11.06.0 → 11.12.0

data/lib/dcmgr/node_modules/sta_collector.rb

@@ -21,6 +21,15 @@ module Dcmgr
       terminate_hook do
       end
 
+      # collect all volume instances on the sta node.
+      def get_available_volumes(sta_node_id)
+        stnode = Models::StorageNode.filter(:node_id=>sta_node_id).first || raise("Unknown sta node ID: #{sta_node_id}")
+        v = stnode.volumes_dataset.lives.all
+        v.map { |volume|
+          volume.merge_pool_data
+        }
+      end
+
       def get_volume(volume_id)
         v = Dcmgr::Models::Volume[volume_id]
         v.merge_pool_data
@@ -34,6 +43,12 @@ module Dcmgr
       def update_volume(volume_id, data)
         v = Dcmgr::Models::Volume[volume_id]
         v.set(data).save
+        if data[:state] == :deleted
+          # Volume#destroy do not really delete row.
+          # just for chain react destroy hooks in the associated models.
+          v.destroy
+        end
+
         # do not respond model object.
         nil
       end
@@ -41,6 +56,11 @@ module Dcmgr
       def update_snapshot(snapshot_id, data)
         vs = Dcmgr::Models::VolumeSnapshot[snapshot_id]
         vs.set(data).save
+        if data[:state] == :deleted
+          # VolumeSnapshot#destroy do not really delete row.
+          # just for chain react destroy hooks in the associated models.
+          vs.destroy
+        end
         # do not respond model object.
         nil
       end

data/lib/dcmgr/node_modules/sta_tgt_initializer.rb

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+require 'isono'
+
+module Dcmgr
+  module NodeModules
+    class StaTgtInitializer < Isono::NodeModules::Base
+      include Dcmgr::Helpers::CliHelper
+      include Dcmgr::Logger
+
+      initialize_hook do
+        
+        manifest.config.initiator_address ||= 'ALL'
+
+        EM.defer do
+          myinstance.register_volumes
+        end
+      end
+      
+      terminate_hook do
+      end
+
+      def register_volumes
+        volumes = rpc.request('sta-collector', 'get_available_volumes', node.node_id)
+        iscsit = Dcmgr::Drivers::IscsiTarget.select_iscsi_target(self.node.manifest.config.iscsi_target, self.node)
+        volumes.each { |volume|
+          iscsit.register(volume)
+        }
+      end
+      
+      def rpc
+        @rpc ||= Isono::NodeModules::RpcChannel.new(@node)
+      end 
+    end 
+  end
+end

data/lib/dcmgr/rack/request_logger.rb

@@ -1,11 +1,14 @@
 # -*- coding: utf-8 -*-
 
+require 'rack/request'
+
 module Dcmgr::Rack
   # Rack middleware for logging each API request.
   class RequestLogger
     HTTP_X_VDC_REQUEST_ID='HTTP_X_VDC_REQUEST_ID'.freeze
     HEADER_X_VDC_REQUEST_ID='X-VDC-Request-ID'.freeze
-    
+    RACK_REQUEST_LOG_KEY='vdc.request_log'.freeze
+      
     def initialize(app, with_header=true)
       raise TypeError unless app.is_a?(Dcmgr::Endpoints::CoreAPI)
       @app = app
@@ -17,8 +20,9 @@ module Dcmgr::Rack
     end
     
     def _call(env)
-      @log = Dcmgr::Models::RequestLog.new
-      log_env(env)
+      request = ::Rack::Request.new(env)
+      env[RACK_REQUEST_LOG_KEY] = @log = Dcmgr::Models::RequestLog.new
+      log_env(request)
       begin
         ret = @app.call(env)
         @log.response_status = ret[0]
@@ -42,8 +46,9 @@ module Dcmgr::Rack
 
     private
     # set common values in Rack env.
-    # @params [Hash] env
-    def log_env(env)
+    # @params [Rack::Request] request
+    def log_env(request)
+      env = request.env
       #@log.frontend_system_id = env[Dcmgr::Endpoints::RACK_FRONTEND_SYSTEM_ID].to_s
       if env[Dcmgr::Endpoints::HTTP_X_VDC_ACCOUNT_UUID].nil? || env[Dcmgr::Endpoints::HTTP_X_VDC_ACCOUNT_UUID] == ''
         @log.account_id = 'nil'
@@ -53,7 +58,7 @@ module Dcmgr::Rack
       @log.requester_token = env[Dcmgr::Endpoints::HTTP_X_VDC_REQUESTER_TOKEN]
       @log.request_method = env['REQUEST_METHOD']
       @log.api_path = env['PATH_INFO']
-      @log.params = ''
+      @log.params = request.params
     end
 
   end

data/lib/dcmgr/rpc/hva_handler.rb

@@ -1,34 +1,13 @@
 # -*- coding: utf-8 -*-
 require 'isono'
-require 'net/telnet'
 require 'fileutils'
+require 'ipaddress'
 
 module Dcmgr
   module Rpc
-    module KvmHelper
-      # Establish telnet connection to KVM monitor console
-      def connect_monitor(port, &blk)
-        begin
-          telnet = ::Net::Telnet.new("Host" => "localhost",
-                                     "Port"=>port.to_s,
-                                     "Prompt" => /\n\(qemu\) \z/,
-                                     "Timeout" => 60,
-                                     "Waittime" => 0.2)
-
-          blk.call(telnet)
-        rescue => e
-          logger.error(e) if self.respond_to?(:logger)
-          raise e
-        ensure
-          telnet.close
-        end
-      end
-    end
-
     class HvaHandler < EndpointBuilder
       include Dcmgr::Logger
       include Dcmgr::Helpers::CliHelper
-      include KvmHelper
       include Dcmgr::Helpers::NicHelper
 
       def select_hypervisor
@@ -43,21 +22,28 @@ module Dcmgr
         tryagain do
           next true if File.exist?(@os_devpath)
 
-          sh("iscsiadm -m discovery -t sendtargets -p %s", [@vol[:storage_pool][:ipaddr]])
+          sh("iscsiadm -m discovery -t sendtargets -p %s", [@vol[:storage_node][:ipaddr]])
           sh("iscsiadm -m node -l -T '%s' --portal '%s'",
-             [@vol[:transport_information][:iqn], @vol[:storage_pool][:ipaddr]])
-          sleep 1
+             [@vol[:transport_information][:iqn], @vol[:storage_node][:ipaddr]])
+          # wait udev queue
+          sh("/sbin/udevadm settle")
         end
 
         rpc.request('sta-collector', 'update_volume', @vol_id, {
                       :state=>:attaching,
                       :attached_at => nil,
+                      :instance_id => @inst[:id], # needed after cleanup
                       :host_device_name => @os_devpath})
       end
 
       def detach_volume_from_host
         # iscsi logout
         sh("iscsiadm -m node -T '%s' --logout", [@vol[:transport_information][:iqn]])
+        # wait udev queue
+        sh("/sbin/udevadm settle")
+      end
+
+      def update_volume_state_to_available
         rpc.request('sta-collector', 'update_volume', @vol_id, {
                       :state=>:available,
                       :host_device_name=>nil,
@@ -67,10 +53,24 @@ module Dcmgr
         event.publish('hva/volume_detached', :args=>[@inst_id, @vol_id])
       end
 
-      def terminate_instance
+      def terminate_instance(state_update=false)
         @hv.terminate_instance(HvaContext.new(self))
+
+        unless @inst[:volume].nil?
+          @inst[:volume].each { |volid, v|
+            @vol_id = volid
+            @vol = v
+            # force to continue detaching volumes during termination.
+            detach_volume_from_host rescue logger.error($!)
+            if state_update
+              update_volume_state_to_available rescue logger.error($!)
+            end
+          }
+        end
+        
+        # cleanup vm data folder
+        FileUtils.rm_r(File.expand_path("#{@inst_id}", @node.manifest.config.vm_data_dir))
       end
-      
 
       def update_instance_state(opts, ev)
         raise "Can't update instance info without setting @inst_id" if @inst_id.nil?
@@ -85,83 +85,209 @@ module Dcmgr
       end
 
       def check_interface
-        vnic = @inst[:instance_nics].first
-        unless vnic.nil?
-          network_map = rpc.request('hva-collector', 'get_network', @inst[:instance_nics].first[:network_id])
-
-          # physical interface
-          physical_if = find_nic(@node.manifest.config.hv_ifindex)
-          raise "UnknownPhysicalNIC" if physical_if.nil?
-
-          if network_map[:vlan_id] == 0
-            # bridge interface
-            bridge_if = @node.manifest.config.bridge_novlan
-            unless valid_nic?(bridge_if)
-              sh("/usr/sbin/brctl addbr %s",    [bridge_if])
-              sh("/usr/sbin/brctl addif %s %s", [bridge_if, physical_if])
-            end
-          else
-            # vlan interface
-            vlan_if = "#{physical_if}.#{network_map[:vlan_id]}"
+        @inst[:instance_nics].each { |vnic|
+          network = rpc.request('hva-collector', 'get_network', vnic[:network_id])
+          
+          fwd_if = phy_if = network[:physical_network][:interface]
+          bridge_if = network[:link_interface]
+          
+          if network[:vlan_id].to_i > 0 && phy_if
+            fwd_if = "#{phy_if}.#{network[:vlan_id]}"
             unless valid_nic?(vlan_if)
-              sh("/sbin/vconfig add #{physical_if} #{network_map[:vlan_id]}")
-            end
-
-            # bridge interface
-            bridge_if = "#{@node.manifest.config.bridge_prefix}-#{physical_if}.#{network_map[:vlan_id]}"
-            unless valid_nic?(bridge_if)
-              sh("/usr/sbin/brctl addbr %s",    [bridge_if])
-              sh("/usr/sbin/brctl addif %s %s", [bridge_if, vlan_if])
+              sh("/sbin/vconfig add #{phy_if} #{network[:vlan_id]}")
+              sh("/sbin/ip link set %s up", [fwd_if])
+              sh("/sbin/ip link set %s promisc on", [fwd_if])
             end
           end
 
-          # interface up? down?
-          [ vlan_if, bridge_if ].each do |ifname|
-            if nic_state(ifname) == "down"
-              sh("/sbin/ifconfig #{ifname} 0.0.0.0 up")
+          unless valid_nic?(bridge_if)
+            sh("/usr/sbin/brctl addbr %s",    [bridge_if])
+            sh("/usr/sbin/brctl setfd %s 0",    [bridge_if])
+            # There is null case for the forward interface to create closed bridge network.
+            if fwd_if
+              sh("/usr/sbin/brctl addif %s %s", [bridge_if, fwd_if])
             end
           end
-          sleep 1
-          bridge_if
-        end
+        }
+        sleep 1
       end
 
 
       def get_linux_dev_path
         # check under until the dev file is created.
         # /dev/disk/by-path/ip-192.168.1.21:3260-iscsi-iqn.1986-03.com.sun:02:a1024afa-775b-65cf-b5b0-aa17f3476bfc-lun-0
-        @os_devpath = "/dev/disk/by-path/ip-%s-iscsi-%s-lun-%d" % ["#{@vol[:storage_pool][:ipaddr]}:3260",
+        @os_devpath = "/dev/disk/by-path/ip-%s-iscsi-%s-lun-%d" % ["#{@vol[:storage_node][:ipaddr]}:3260",
                                                                       @vol[:transport_information][:iqn],
                                                                       @vol[:transport_information][:lun]]
       end
 
+      def setup_metadata_drive
+        logger.info("Setting up metadata drive image for :#{@hva_ctx.inst_id}")
+        # truncate creates sparsed file.
+        sh("/usr/bin/truncate -s 10m '#{@hva_ctx.metadata_img_path}'; sync;")
+        # TODO: need to lock loop device not to use same device from
+        # another thread/process.
+        lodev=`/sbin/losetup -f`.chomp
+        sh("/sbin/losetup #{lodev} '#{@hva_ctx.metadata_img_path}'")
+        sh("mkfs.vfat -n METADATA '#{@hva_ctx.metadata_img_path}'")
+        Dir.mkdir("#{@hva_ctx.inst_data_dir}/tmp") unless File.exists?("#{@hva_ctx.inst_data_dir}/tmp")
+        sh("/bin/mount -t vfat #{lodev} '#{@hva_ctx.inst_data_dir}/tmp'")
+
+        vnic = @inst[:instance_nics].first || {}
+        # Appendix B: Metadata Categories
+        # http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/index.html?AESDG-chapter-instancedata.html
+        metadata_items = {
+          'ami-id' => @inst[:image][:uuid],
+          'ami-launch-index' => 0,
+          'ami-manifest-path' => nil,
+          'ancestor-ami-ids' => nil,
+          'block-device-mapping/root' => '/dev/sda',
+          'hostname' => @inst[:hostname],
+          'instance-action' => @inst[:state],
+          'instance-id' => @inst[:uuid],
+          'instance-type' => @inst[:instance_spec][:uuid],
+          'kernel-id' => nil,
+          'local-hostname' => @inst[:hostname],
+          'local-ipv4' => @inst[:ips].first,
+          'mac' => vnic[:mac_addr].unpack('A2'*6).join(':'),
+          'placement/availability-zone' => nil,
+          'product-codes' => nil,
+          'public-hostname' => @inst[:hostname],
+          'public-ipv4'    => @inst[:nat_ips].first,
+          'ramdisk-id' => nil,
+          'reservation-id' => nil,
+          'security-groups' => @inst[:security_groups].join(' '),
+        }
+
+        @inst[:vif].each { |vnic|
+          netaddr  = IPAddress::IPv4.new("#{vnic[:ipv4][:network][:ipv4_network]}/#{vnic[:ipv4][:network][:prefix]}")
+
+          # vfat doesn't allow folder name including ":".
+          # folder name including mac address replaces "-" to ":".
+          mac = vnic[:mac_addr].unpack('A2'*6).join('-')
+          metadata_items.merge!({
+            "network/interfaces/macs/#{mac}/local-hostname" => @inst[:hostname],
+            "network/interfaces/macs/#{mac}/local-ipv4s" => vnic[:ipv4][:address],
+            "network/interfaces/macs/#{mac}/mac" => vnic[:mac_addr].unpack('A2'*6).join(':'),
+            "network/interfaces/macs/#{mac}/public-hostname" => @inst[:hostname],
+            "network/interfaces/macs/#{mac}/public-ipv4s" => vnic[:ipv4][:nat_address],
+            "network/interfaces/macs/#{mac}/security-groups" => @inst[:security_groups].join(' '),
+            # wakame-vdc extention items.
+            # TODO: need an iface index number?
+            "network/interfaces/macs/#{mac}/x-gateway" => vnic[:ipv4][:network][:ipv4_gw],
+            "network/interfaces/macs/#{mac}/x-netmask" => netaddr.prefix.to_ip,
+            "network/interfaces/macs/#{mac}/x-network" => vnic[:ipv4][:network][:ipv4_network],
+            "network/interfaces/macs/#{mac}/x-broadcast" => netaddr.broadcast,
+            "network/interfaces/macs/#{mac}/x-metric" => vnic[:ipv4][:network][:metric],
+          })
+        }
+
+        if @inst[:ssh_key_data]
+          metadata_items.merge!({
+            "public-keys/0=#{@inst[:ssh_key_data][:name]}" => @inst[:ssh_key_data][:public_key],
+            'public-keys/0/openssh-key'=> @inst[:ssh_key_data][:public_key],
+          })
+        else
+          metadata_items.merge!({'public-keys/'=>nil})
+        end
+
+        # build metadata directory tree
+        metadata_base_dir = File.expand_path("meta-data", "#{@hva_ctx.inst_data_dir}/tmp")
+        FileUtils.mkdir_p(metadata_base_dir)
+        
+        metadata_items.each { |k, v|
+          if k[-1,1] == '/' && v.nil?
+            # just create empty folder
+            FileUtils.mkdir_p(File.expand_path(k, metadata_base_dir))
+            next
+          end
+          
+          dir = File.dirname(k)
+          if dir != '.'
+            FileUtils.mkdir_p(File.expand_path(dir, metadata_base_dir))
+          end
+          File.open(File.expand_path(k, metadata_base_dir), 'w') { |f|
+            f.puts(v.to_s)
+          }
+        }
+        # user-data
+        File.open(File.expand_path('user-data', "#{@hva_ctx.inst_data_dir}/tmp"), 'w') { |f|
+          f.puts(@inst[:user_data])
+        }
+        
+      ensure
+        # ignore any errors from cleanup work.
+        sh("/bin/umount -f '#{@hva_ctx.inst_data_dir}/tmp'") rescue logger.warn($!.message)
+        sh("/sbin/losetup -d #{lodev}") rescue logger.warn($!.message)
+      end
+
       job :run_local_store, proc {
         @inst_id = request.args[0]
         logger.info("Booting #{@inst_id}")
 
         @inst = rpc.request('hva-collector', 'get_instance',  @inst_id)
-        raise "Invalid instance state: #{@inst[:state]}" unless %w(init failingover).member?(@inst[:state].to_s)
+        raise "Invalid instance state: #{@inst[:state]}" unless %w(pending failingover).member?(@inst[:state].to_s)
 
         # select hypervisor :kvm, :lxc
         select_hypervisor
 
         # create hva context
-        hc = HvaContext.new(self)
+        @hva_ctx = HvaContext.new(self)
 
         rpc.request('hva-collector', 'update_instance', @inst_id, {:state=>:starting})
         # setup vm data folder
-        inst_data_dir = hc.inst_data_dir
+        inst_data_dir = @hva_ctx.inst_data_dir
         FileUtils.mkdir(inst_data_dir) unless File.exists?(inst_data_dir)
         # copy image file
         img_src = @inst[:image][:source]
         @os_devpath = File.expand_path("#{@inst[:uuid]}", inst_data_dir)
-        sh("curl --silent -o '#{@os_devpath}' #{img_src[:uri]}")
+
+        # vmimage cache
+        vmimg_cache_dir = File.expand_path("_base", @node.manifest.config.vm_data_dir)
+        FileUtils.mkdir_p(vmimg_cache_dir) unless File.exists?(vmimg_cache_dir)
+        vmimg_basename = File.basename(img_src[:uri])
+        vmimg_cache_path = File.expand_path(vmimg_basename, vmimg_cache_dir)
+
+        logger.debug("preparing #{@os_devpath}")
+
+        # vmimg cached?
+        unless File.exists?(vmimg_cache_path)
+          logger.debug("copying #{img_src[:uri]} to #{vmimg_cache_path}")
+          pararell_curl("#{img_src[:uri]}", "#{vmimg_cache_path}")
+        else
+          md5sum = sh("md5sum #{vmimg_cache_path}")
+          if md5sum[:stdout].split(' ')[0] == @inst[:image][:md5sum]
+            logger.debug("verified vm cache image: #{vmimg_cache_path}")
+          else
+            logger.debug("not verified vm cache image: #{vmimg_cache_path}")
+            sh("rm -f %s", [vmimg_cache_path])
+            tmp_id = Isono::Util::gen_id
+            logger.debug("copying #{img_src[:uri]} to #{vmimg_cache_path}")
+            pararell_curl("#{img_src[:uri]}", "#{vmimg_cache_path}.#{tmp_id}")
+
+            sh("mv #{vmimg_cache_path}.#{tmp_id} #{vmimg_cache_path}")
+            logger.debug("vmimage cache deployed on #{vmimg_cache_path}")
+          end
+        end
+
+        ####
+        logger.debug("copying #{vmimg_cache_path} to #{@os_devpath}")
+        case vmimg_cache_path
+        when /\.gz$/
+          sh("zcat %s | cp --sparse=always /dev/stdin %s",[vmimg_cache_path, @os_devpath])
+        else
+          sh("cp -p --sparse=always %s %s",[vmimg_cache_path, @os_devpath])
+        end
+
         sleep 1
 
-        @bridge_if = check_interface
-        @hv.run_instance(hc)
+        setup_metadata_drive
+        
+        check_interface
+        @hv.run_instance(@hva_ctx)
         update_instance_state({:state=>:running}, 'hva/instance_started')
       }, proc {
+        terminate_instance(false) rescue logger.error($!)
         update_instance_state({:state=>:terminated, :terminated_at=>Time.now.utc},
                               'hva/instance_terminated')
       }
@@ -169,28 +295,23 @@ module Dcmgr
       job :run_vol_store, proc {
         @inst_id = request.args[0]
         @vol_id = request.args[1]
-
         @inst = rpc.request('hva-collector', 'get_instance', @inst_id)
         @vol = rpc.request('sta-collector', 'get_volume', @vol_id)
         logger.info("Booting #{@inst_id}")
-        raise "Invalid instance state: #{@inst[:state]}" unless %w(init failingover).member?(@inst[:state].to_s)
+        raise "Invalid instance state: #{@inst[:state]}" unless %w(pending failingover).member?(@inst[:state].to_s)
 
         # select hypervisor :kvm, :lxc
         select_hypervisor
 
         # create hva context
-        hc = HvaContext.new(self)
+        @hva_ctx = HvaContext.new(self)
 
         rpc.request('hva-collector', 'update_instance', @inst_id, {:state=>:starting})
 
         # setup vm data folder
-        inst_data_dir = hc.inst_data_dir
+        inst_data_dir = @hva_ctx.inst_data_dir
         FileUtils.mkdir(inst_data_dir) unless File.exists?(inst_data_dir)
 
-        # create volume from snapshot
-        jobreq.run("zfs-handle.#{@vol[:storage_pool][:node_id]}", "create_volume", @vol_id)
-
-        logger.debug("volume created on #{@vol[:storage_pool][:node_id]}: #{@vol_id}")
         # reload volume info
         @vol = rpc.request('sta-collector', 'get_volume', @vol_id)
         
@@ -203,14 +324,20 @@ module Dcmgr
         # attach disk
         attach_volume_to_host
         
+        setup_metadata_drive
+        
         # run vm
-        @bridge_if = check_interface
-        @hv.run_instance(HvaContext.new(self))
+        check_interface
+        @hv.run_instance(@hva_ctx)
         update_instance_state({:state=>:running}, 'hva/instance_started')
         update_volume_state({:state=>:attached, :attached_at=>Time.now.utc}, 'hva/volume_attached')
       }, proc {
+        # TODO: Run detach & destroy volume
         update_instance_state({:state=>:terminated, :terminated_at=>Time.now.utc},
                               'hva/instance_terminated')
+        terminate_instance(false) rescue logger.error($!)
+        update_volume_state({:state=>:deleted, :deleted_at=>Time.now.utc},
+                              'hva/volume_deleted')
       }
 
       job :terminate do
@@ -224,28 +351,15 @@ module Dcmgr
 
         begin
           rpc.request('hva-collector', 'update_instance',  @inst_id, {:state=>:shuttingdown})
-
-          terminate_instance
-
-          unless @inst[:volume].nil?
-            @inst[:volume].each { |volid, v|
-              @vol_id = volid
-              @vol = v
-              # force to continue detaching volumes during termination.
-              detach_volume_from_host rescue logger.error($!)
-            }
-          end
-
-          # cleanup vm data folder
-          FileUtils.rm_r(File.expand_path("#{@inst_id}", @node.manifest.config.vm_data_dir))
+          terminate_instance(true)
         ensure
           update_instance_state({:state=>:terminated,:terminated_at=>Time.now.utc},
                                 'hva/instance_terminated')
         end
       end
 
-      # just do terminate instance and unmount volumes not to affect
-      # state management.
+      # just do terminate instance and unmount volumes. it should not change
+      # state on any resources.
       # called from HA at which the faluty instance get cleaned properly.
       job :cleanup do
         @inst_id = request.args[0]
@@ -253,19 +367,36 @@ module Dcmgr
         @inst = rpc.request('hva-collector', 'get_instance', @inst_id)
         raise "Invalid instance state: #{@inst[:state]}" unless @inst[:state].to_s == 'running'
 
+        # select hypervisor :kvm, :lxc
+        select_hypervisor
+
         begin
-          terminate_instance
-
-          unless @inst[:volume].nil?
-            @inst[:volume].each { |volid, v|
-              @vol_id = volid
-              @vol = v
-              # force to continue detaching volumes during termination.
-              detach_volume_from_host rescue logger.error($!)
-            }
-          end
+          terminate_instance(false)
+        ensure
+          # just publish "hva/instance_terminated" to update security group rules once
+          update_instance_state({}, 'hva/instance_terminated')
         end
+      end
+
+      # stop instance is mostly similar to terminate_instance. the
+      # difference is the state transition of instance and associated
+      # resources to the instance , attached volumes and vnic, are kept
+      # same sate.
+      job :stop do
+        @inst_id = request.args[0]
 
+        @inst = rpc.request('hva-collector', 'get_instance', @inst_id)
+        raise "Invalid instance state: #{@inst[:state]}" unless @inst[:state].to_s == 'running'
+
+        select_hypervisor
+
+        begin
+          rpc.request('hva-collector', 'update_instance',  @inst_id, {:state=>:stopping})
+          terminate_instance(false)
+        ensure
+          # 
+          update_instance_state({:state=>:stopped, :host_node_id=>nil}, 'hva/instance_terminated')
+        end
       end
 
       job :attach, proc {
@@ -291,7 +422,11 @@ module Dcmgr
         logger.info("Attaching #{@vol_id} on #{@inst_id}")
 
         # attach disk on guest os
-        pci_devaddr = @hv.attach_volume_to_guest(HvaContext.new(self))
+        pci_devaddr=nil
+        tryagain do
+          pci_devaddr = @hv.attach_volume_to_guest(HvaContext.new(self))
+        end
+        raise "Can't attach #{@vol_id} on #{@inst_id}" if pci_devaddr.nil?
 
         rpc.request('sta-collector', 'update_volume', @vol_id, {
                       :state=>:attached,
@@ -299,6 +434,12 @@ module Dcmgr
                       :guest_device_name=>pci_devaddr})
         event.publish('hva/volume_attached', :args=>[@inst_id, @vol_id])
         logger.info("Attached #{@vol_id} on #{@inst_id}")
+      }, proc {
+        # TODO: Run detach volume
+        # push back volume state to available.
+        update_volume_state({:state=>:available},
+                            'hva/volume_available')
+        logger.error("Attach failed: #{@vol_id} on #{@inst_id}")
       }
 
       job :detach do
@@ -315,10 +456,13 @@ module Dcmgr
 
         rpc.request('sta-collector', 'update_volume', @vol_id, {:state=>:detaching, :detached_at=>nil})
         # detach disk on guest os
-        @hv.detach_volume_from_guest(HvaContext.new(self))
+        tryagain do
+          @hv.detach_volume_from_guest(HvaContext.new(self))
+        end
 
         # detach disk on host os
         detach_volume_from_host
+        update_volume_state_to_available
       end
 
       job :reboot, proc {
@@ -328,9 +472,6 @@ module Dcmgr
         # select_hypervisor :kvm, :lxc
         select_hypervisor
 
-        # check interface
-        @bridge_if = check_interface
-
         # reboot instance
         @hv.reboot_instance(HvaContext.new(self))
       }
@@ -346,6 +487,11 @@ module Dcmgr
       def event
         @event ||= Isono::NodeModules::EventChannel.new(@node)
       end
+
+      def pararell_curl(url, output_path)
+        script_root_path = File.join(File.expand_path('../../../../',__FILE__), 'script')
+        sh("#{script_root_path}/pararell-curl.sh --url=#{url} --output_path=#{output_path}")
+      end
     end
 
     class HvaContext
@@ -371,8 +517,8 @@ module Dcmgr
         @hva.instance_variable_get(:@os_devpath)
       end
 
-      def bridge_if
-        @hva.instance_variable_get(:@bridge_if)
+      def metadata_img_path
+        File.expand_path('metadata.img', inst_data_dir)
       end
 
       def vol