bosh_warden_cpi 1.5.0.pre.3 → 1.2513.0

data/lib/cloud/warden/diskutils.rb

@@ -0,0 +1,107 @@
+module Bosh::WardenCloud
+  class DiskUtils
+
+    include Bosh::WardenCloud::Helpers
+
+    UMOUNT_GUARD_RETRIES = 60
+    UMOUNT_GUARD_SLEEP = 3
+
+    def initialize(disk_root, stemcell_root, fs_type)
+      @disk_root = disk_root
+      @fs_type = fs_type
+      @stemcell_root = stemcell_root
+
+      unless Dir.exist?(disk_root)
+        FileUtils.mkdir_p(disk_root)
+      end
+    end
+
+    def create_disk(disk_id, size)
+      raise ArgumentError, 'disk size <= 0' unless size > 0
+
+      image_file = image_path(disk_id)
+      FileUtils.touch(image_file)
+      File.truncate(image_file, size << 20) # 1 MB == 1<<20 Byte
+      sh "/sbin/mkfs -t #{@fs_type} -F #{image_file} 2>&1"
+
+    rescue => e
+      if image_file
+        FileUtils.rm_f image_file if File.exist?(image_file)
+      end
+      raise e
+    end
+
+    def delete_disk(disk_id)
+      FileUtils.rm_f image_path(disk_id)
+    end
+
+    def disk_exist?(disk_id)
+      File.exist?(image_path(disk_id))
+    end
+
+    def mount_disk(path, disk_id)
+      unless Dir.exist?(path)
+        FileUtils.mkdir_p(path)
+      end
+
+      disk_img = image_path(disk_id)
+      sudo "mount #{disk_img} #{path} -o loop"
+    end
+
+    def umount_disk(path)
+      umount_guard path
+    end
+
+    def stemcell_unpack(image_path, stemcell_id)
+      stemcell_dir = stemcell_path(stemcell_id)
+      unless Dir.exist?(stemcell_dir)
+        FileUtils.mkdir_p(stemcell_dir)
+      end
+      raise "#{image_path} not exist for creating stemcell" unless File.exist?(image_path)
+      sudo "tar -C #{stemcell_dir} -xzf #{image_path} 2>&1"
+    rescue => e
+      sudo "rm -rf #{stemcell_dir}"
+      raise e
+    end
+
+    def stemcell_delete(stemcell_id)
+      stemcell_dir = stemcell_path(stemcell_id)
+      sudo "rm -rf #{stemcell_dir}"
+    end
+
+    def image_path(disk_id)
+      File.join(@disk_root, "#{disk_id}.img")
+    end
+
+    def stemcell_path(stemcell_id)
+      File.join(@stemcell_root, stemcell_id)
+    end
+
+    private
+
+    def mount_entry(partition)
+      `mount`.lines.select { |l| l.match(/#{partition}/) }.first
+    end
+
+    # Retry the umount for GUARD_RETRIES +1  times
+    def umount_guard(mountpoint)
+      umount_attempts = UMOUNT_GUARD_RETRIES
+
+      loop do
+        return if mount_entry(mountpoint).nil?
+        sudo "umount #{mountpoint}" do |result|
+          if result.success?
+            return
+          elsif umount_attempts != 0
+            sleep UMOUNT_GUARD_SLEEP
+            umount_attempts -= 1
+          else
+            raise "Failed to umount #{mountpoint}: #{result.output}"
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/cloud/warden/helpers.rb

@@ -2,58 +2,105 @@ module Bosh::WardenCloud
 
   module Helpers
 
+    DEFAULT_SETTINGS_FILE = '/var/vcap/bosh/settings.json'
+
     def cloud_error(message)
       @logger.error(message) if @logger
       raise Bosh::Clouds::CloudError, message
     end
 
-    def uuid(klass=nil)
+    def uuid(klass = nil)
       id = SecureRandom.uuid
-
       if klass
-        id = "%s-%s" % [klass, id]
+        id = sprintf('%s-%s', klass, id)
       end
-
       id
     end
 
-
     def sudo(cmd)
-      logger.info "run 'sudo -n #{cmd}'"
-      Bosh::Exec.sh "sudo -n #{cmd}"
+      sh(cmd, true)
+    end
+
+    def sh(cmd, su = false)
+      runcmd = su == true ? "sudo -n #{cmd}" : cmd
+      @logger.info "run '#{runcmd}'" if @logger
+      Bosh::Exec.sh("#{runcmd}", yield: :on_false) do |result|
+        yield result if block_given?
+      end
+    end
+
+    def with_warden
+      client = Warden::Client.new(@warden_unix_path)
+      client.connect
+      ret = yield client
+      ret
+    ensure
+      client.disconnect if client
+    end
+
+    def agent_settings_file
+      DEFAULT_SETTINGS_FILE
     end
 
-    def sh(cmd)
-      logger.info "run '#{cmd}'"
-      Bosh::Exec.sh "#{cmd}"
+    def generate_agent_env(vm_id, agent_id, networks, environment)
+      vm_env = {
+          'name' => vm_id,
+          'id' => vm_id
+      }
+
+      env = {
+          'vm' => vm_env,
+          'agent_id' => agent_id,
+          'networks' => networks,
+          'disks' => { 'persistent' => {} },
+      }
+      env['env'] = environment if environment
+      env.merge!(@agent_properties)
+      env
     end
 
-    ##
-    # This method generates a script that is run inside a container, to get an
-    # available device path.
-    #
-    # This is hacky. The attached device is already formatted. In order to trick
-    # bosh agent not to format the disk again, we touch an empty device file and
-    # mknod the real partition file that is already formatted. Bosh agent will
-    # mount skip the format process and directly mount the partition file.
-    #
-    # e.g.
-    # Device file is like /dev/sda
-    # Partition file is like /dev/sda1
-    def attach_script(device_number, device_prefix)
-      script = <<-EOF
-for i in a b c d e f g h; do (stat #{device_prefix}${i} > /dev/null 2>&1) || break; done
-touch #{device_prefix}${i}
-mknod #{device_prefix}${i}1 b 7 #{device_number} > /dev/null 2>&1 && echo "#{device_prefix}${i}"
-      EOF
+    def get_agent_env(handle)
+      body = with_warden do |client|
+        request = Warden::Protocol::RunRequest.new
+        request.handle = handle
+        request.privileged = true
+        request.script = "cat #{agent_settings_file}"
+        client.call(request).stdout
+      end
+      env = Yajl::Parser.parse(body)
+      env
     end
 
-    def partition_path(device_path)
-      "#{device_path}1"
+    def set_agent_env(handle, env)
+      tempfile = File.new("/tmp/agent-setting-#{Time.now.to_f}-#{Kernel.rand(100_000)}",'w')
+      tempfile.write(Yajl::Encoder.encode(env))
+      tempfile_in = "/tmp/#{Kernel.rand(100_000)}"
+      tempfile.close
+      # Here we copy the setting file to temp file in container, then mv it to
+      # /var/vcap/bosh by privileged user.
+      with_warden do |client|
+        request = Warden::Protocol::CopyInRequest.new
+        request.handle = handle
+        request.src_path = tempfile.path
+        request.dst_path = tempfile_in
+        client.call(request)
+
+        request = Warden::Protocol::RunRequest.new
+        request.handle = handle
+        request.privileged = true
+        request.script = "mv #{tempfile_in} #{agent_settings_file}"
+        client.call(request)
+      end
     end
 
-    def process_user
-      Etc.getpwuid(Process.uid).name
+    def start_agent(handle)
+      with_warden do |client|
+        request = Warden::Protocol::SpawnRequest.new
+        request.handle = handle
+        request.privileged = true
+        request.script = '/usr/sbin/runsvdir-start'
+        client.call(request)
+      end
     end
 
   end

data/lib/cloud/warden/version.rb

@@ -1,7 +1,5 @@
 module Bosh
-  module Clouds
-    class Warden
-      VERSION = '1.5.0.pre.3'
-    end
+  module WardenCloud
+    VERSION = '1.2513.0'
   end
 end

data/lib/cloud/warden.rb

@@ -1,40 +1,22 @@
-require "yajl"
-require "sequel"
-require "fileutils"
-require "tempfile"
-require "securerandom"
-require "etc"
+require 'yajl'
+require 'fileutils'
+require 'tempfile'
+require 'securerandom'
 
-require "common/exec"
-require "common/thread_pool"
-require "common/thread_formatter"
+require 'common/exec'
+require 'common/thread_pool'
+require 'common/thread_formatter'
 
-require "cloud"
-require "cloud/warden/helpers"
-require "cloud/warden/device_pool"
-require "cloud/warden/cloud"
-require "cloud/warden/version"
-require "cloud/warden/models/vm"
-require "cloud/warden/models/disk"
+require 'cloud'
+require 'cloud/warden/helpers'
+require 'cloud/warden/cloud'
+require 'cloud/warden/diskutils'
+require 'cloud/warden/version'
 
-require "warden/client"
+require 'warden/client'
 
 module Bosh
   module Clouds
-    class Warden
-      extend Forwardable
-
-      def_delegators :@delegate,
-                     :create_stemcell, :delete_stemcell,
-                     :create_vm, :delete_vm, :reboot_vm,
-                     :configure_networks,
-                     :create_disk, :delete_disk,
-                     :attach_disk, :detach_disk,
-                     :validate_deployment
-
-      def initialize(options)
-        @delegate = WardenCloud::Cloud.new(options)
-      end
-    end
+    Warden = Bosh::WardenCloud::Cloud
   end
 end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+require 'rspec'
+require 'logger'
+require 'tmpdir'
+require 'cloud'
+require 'cloud/warden'
+
+def asset(file)
+  File.join(File.dirname(__FILE__), 'assets', file)
+end
+
+def mock_sh (cmd, su = false, times = 1, success = true)
+  zero_exit_status = double('Process::Status', exit_status: 0)
+  result = double('Result', :success? => success)
+  prefix = (su == true)? 'sudo -n ' : ''
+  expect(Bosh::Exec).to receive(:sh).exactly(times).times.with(/#{prefix}#{cmd}/, yield: :on_false).and_yield(result).and_return(zero_exit_status)
+end
+
+RSpec.configure do |conf|
+  conf.before(:each) { allow(Bosh::Clouds::Config).to receive(:logger).and_return(double.as_null_object)  }
+end

data/spec/unit/cloud_spec.rb

@@ -0,0 +1,264 @@
+require 'spec_helper'
+
+describe Bosh::WardenCloud::Cloud do
+
+  DEFAULT_HANDLE = 'vm-uuid-1234'
+  DEFAULT_AGENT_ID = 'agent-abcd'
+  DEFAULT_STEMCELL_ID = 'stemcell-uuid'
+
+  before :each do
+    @logger = Bosh::Clouds::Config.logger
+    @disk_root = Dir.mktmpdir('warden-cpi-disk')
+    @stemcell_path =  Dir.mktmpdir('stemcell-disk')
+    @stemcell_root = File.join(@stemcell_path, DEFAULT_STEMCELL_ID)
+    @disk_util = double('DiskUtils')
+    @warden_client = double(Warden::Client)
+    allow(@disk_util).to receive(:stemcell_path).and_return(@stemcell_root)
+    allow(Bosh::WardenCloud::DiskUtils).to receive(:new).with(@disk_root, @stemcell_path, 'ext4').and_return(@disk_util)
+    allow(Warden::Client).to receive(:new).and_return(@warden_client)
+
+    cloud_options = {
+        'disk' => {
+            'root' => @disk_root,
+            'fs' => 'ext4',
+        },
+        'stemcell' => {
+            'root' => @stemcell_path,
+        },
+        'agent' => {
+            'mbus' => 'nats://nats:nats@192.168.50.4:21084',
+            'blobstore' => {
+                'provider' => 'simple',
+                'options' => 'option'
+            }                          ,
+            'ntp' => []
+        }
+    }
+    @cloud = Bosh::Clouds::Provider.create(:warden, cloud_options)
+
+    allow(@warden_client).to receive(:connect) {}
+    allow(@warden_client).to receive(:disconnect) {}
+  end
+
+  after :each do
+    FileUtils.rm_rf @stemcell_path
+    FileUtils.rm_rf @disk_root
+  end
+
+  context 'initialize' do
+    it 'can be created using Bosh::Clouds::Provider' do
+      expect(@cloud).to be_an_instance_of(Bosh::Clouds::Warden)
+    end
+  end
+
+  context 'create_vm' do
+    before :each do
+      allow(@cloud).to receive(:uuid).with('vm') { DEFAULT_HANDLE }
+      Dir.mkdir(@stemcell_root)
+      allow(@warden_client).to receive(:call) do |req|
+        res = req.create_response
+        case req
+          when Warden::Protocol::CreateRequest
+            expect(req.network).to eq('1.1.1.1')
+            expect(req.rootfs).to equal(@stemcell_root)
+            expect(req.bind_mounts[0].src_path).to match(/#{@disk_root}\/bind_mount_points/)
+            expect(req.bind_mounts[1].src_path).to match(/#{@disk_root}\/ephemeral_mount_point/)
+            expect(req.bind_mounts[0].dst_path).to eq('/warden-cpi-dev')
+            expect(req.bind_mounts[1].dst_path).to eq('/var/vcap/data')
+            expect(req.bind_mounts[0].mode).to eq(Warden::Protocol::CreateRequest::BindMount::Mode::RW)
+            expect(req.bind_mounts[1].mode).to eq(Warden::Protocol::CreateRequest::BindMount::Mode::RW)
+            res.handle = DEFAULT_HANDLE
+          when Warden::Protocol::CopyInRequest
+            raise 'Container not found' unless req.handle == DEFAULT_HANDLE
+            env = Yajl::Parser.parse(File.read(req.src_path))
+            expect(env['agent_id']).to eq(DEFAULT_AGENT_ID)
+            expect(env['vm']['name']).not_to be_nil
+            expect(env['vm']['id']).not_to be_nil
+            expect(env['mbus']).not_to be_nil
+            expect(env['ntp']).to be_an_instance_of Array
+            expect(env['blobstore']).to be_an_instance_of Hash
+          when Warden::Protocol::RunRequest
+            # Ignore
+          when Warden::Protocol::SpawnRequest
+            expect(req.script).to eq('/usr/sbin/runsvdir-start')
+            expect(req.privileged).to be true
+          when Warden::Protocol::DestroyRequest
+            expect(req.handle).to eq(DEFAULT_HANDLE)
+            @destroy_called = true
+          else
+            raise "#{req} not supported"
+        end
+        res
+      end
+    end
+
+    it 'can create vm' do
+      expect(@cloud).to receive(:sudo).exactly(3)
+      network_spec = {
+          'nic1' => { 'ip' => '1.1.1.1', 'type' => 'static' },
+      }
+      @cloud.create_vm(DEFAULT_AGENT_ID, DEFAULT_STEMCELL_ID, nil, network_spec)
+    end
+
+    it 'should raise error for invalid stemcell' do
+      allow(@disk_util).to receive(:stemcell_path).and_return('invalid_dir')
+      expect {
+        @cloud.create_vm('agent_id', 'invalid_stemcell_id', nil, {})
+      }.to raise_error Bosh::Clouds::CloudError
+    end
+
+    it 'should raise error for more than 1 nics' do
+      expect {
+        network_spec = {
+            'nic1' => { 'ip' => '1.1.1.1', 'type' => 'static' },
+            'nic2' => { 'type' => 'dynamic' },
+        }
+        @cloud.create_vm('agent_id', 'invalid_stemcell_id', nil, network_spec)
+      }.to raise_error ArgumentError
+    end
+
+    it 'should clean up when an error raised' do
+      class FakeError < StandardError; end
+      allow(@cloud).to receive(:sudo) {}
+      allow(@cloud).to receive(:set_agent_env) { raise FakeError.new }
+      network_spec = {
+          'nic1' => { 'ip' => '1.1.1.1', 'type' => 'static' },
+      }
+      begin
+        @cloud.create_vm(DEFAULT_AGENT_ID, DEFAULT_STEMCELL_ID, nil, network_spec)
+      rescue FakeError
+      else
+        raise 'Expected FakeError'
+      end
+      expect(@destroy_called).to be true
+    end
+  end
+
+  context 'delete_vm' do
+    it 'can delete vm' do
+      allow(@warden_client).to receive(:call) do |req|
+        res = req.create_response
+        case req
+          when Warden::Protocol::DestroyRequest
+            expect(req.handle).to eq(DEFAULT_HANDLE)
+          when Warden::Protocol::ListRequest
+            res.handles = DEFAULT_HANDLE
+          else
+            raise "#{req} not supported"
+        end
+        res
+      end
+      mock_sh("umount #{@disk_root}/bind_mount_points/#{DEFAULT_HANDLE}", true)
+      mock_sh("rm -rf #{@disk_root}/ephemeral_mount_point/#{DEFAULT_HANDLE}", true)
+      mock_sh("rm -rf #{@disk_root}/bind_mount_points/#{DEFAULT_HANDLE}", true)
+      @cloud.delete_vm(DEFAULT_HANDLE)
+    end
+
+    it 'should proceed even delete a vm which not exist' do
+      allow(@cloud).to receive(:has_vm?).with('vm_not_existed').and_return(false)
+      mock_sh("rm -rf #{@disk_root}/ephemeral_mount_point/vm_not_existed", true)
+      mock_sh("rm -rf #{@disk_root}/bind_mount_points/vm_not_existed", true)
+      expect {
+        @cloud.delete_vm('vm_not_existed')
+      }.to_not raise_error
+    end
+
+    it 'can delete a vm with disk attached' do
+      allow(@warden_client).to receive(:call) do |req|
+        res = req.create_response
+        case req
+          when Warden::Protocol::DestroyRequest
+            expect(req.handle).to eq(DEFAULT_HANDLE)
+          when Warden::Protocol::ListRequest
+            res.handles = DEFAULT_HANDLE
+          else
+            raise "#{req} not supported"
+        end
+        res
+      end
+      mock_sh("umount #{@disk_root}/bind_mount_points/#{DEFAULT_HANDLE}", true)
+      mock_sh("rm -rf #{@disk_root}/ephemeral_mount_point/#{DEFAULT_HANDLE}", true)
+      mock_sh("rm -rf #{@disk_root}/bind_mount_points/#{DEFAULT_HANDLE}", true)
+      @cloud.delete_vm(DEFAULT_HANDLE)
+    end
+  end
+
+  context 'has_vm' do
+    before :each do
+      allow(@warden_client).to receive(:call) do |req|
+        res = req.create_response
+        case req
+          when Warden::Protocol::ListRequest
+            res.handles = DEFAULT_HANDLE
+          else
+            raise "#{req} not supported"
+        end
+        res
+      end
+    end
+
+    it 'return true when container exist' do
+      expect(@cloud.has_vm?(DEFAULT_HANDLE)).to be true
+    end
+
+    it 'return false when container not exist' do
+      expect(@cloud.has_vm?('vm_not_exist')).to be false
+    end
+  end
+
+  context 'stemcells' do
+    before :each do
+      allow(@cloud).to receive(:uuid).with('stemcell') { DEFAULT_STEMCELL_ID }
+    end
+
+    it 'invoke disk_utils to create stemcell with uuid' do
+      expect(@disk_util).to receive(:stemcell_unpack).with('imgpath', DEFAULT_STEMCELL_ID)
+      @cloud.create_stemcell('imgpath', nil)
+    end
+
+    it 'invoke disk_utils to delete stemcell with uuid' do
+      expect(@disk_util).to receive(:stemcell_delete).with(DEFAULT_STEMCELL_ID)
+      @cloud.delete_stemcell(DEFAULT_STEMCELL_ID)
+    end
+  end
+
+  context 'disk create/delete/attach/detach' do
+    before :each do
+      allow(@cloud).to receive(:uuid).with('disk') { 'disk-uuid-1234' }
+    end
+
+    it 'invoke disk_utils to create disk with uuid' do
+      expect(@disk_util).to receive(:create_disk).with('disk-uuid-1234', 1024)
+      @cloud.create_disk(1024, nil)
+    end
+
+    it 'invoke disk_utils to delete disk with uuid' do
+      expect(@disk_util).to receive(:disk_exist?).with('disk-uuid-1234').and_return(true)
+      expect(@disk_util).to receive(:delete_disk).with('disk-uuid-1234')
+      @cloud.delete_disk('disk-uuid-1234')
+    end
+
+    it 'invoke disk_utils to mount disk and setup agent env when attach disk' do
+      allow(@cloud).to receive(:get_agent_env) { { 'disks' => { 'persistent' => {} } } }
+      expected_env = { 'disks' => { 'persistent' => { 'disk-uuid-1234' => '/warden-cpi-dev/disk-uuid-1234' } } }
+      expected_mountpoint = File.join(@disk_root, 'bind_mount_points', 'vm-uuid-1234', 'disk-uuid-1234')
+      expect(@disk_util).to receive(:disk_exist?).with('disk-uuid-1234').and_return(true)
+      expect(@disk_util).to receive(:mount_disk).with(expected_mountpoint, 'disk-uuid-1234')
+      expect(@cloud).to receive(:set_agent_env).with('vm-uuid-1234', expected_env)
+      expect(@cloud).to receive(:has_vm?).with('vm-uuid-1234').and_return(true)
+      @cloud.attach_disk('vm-uuid-1234', 'disk-uuid-1234')
+    end
+
+    it 'invoke disk_utils to umount disk and remove agent env when detach disk' do
+      allow(@cloud).to receive(:get_agent_env) { { 'disks' => { 'persistent' => { 'disk-uuid-1234' => '/warden-cpi-dev/disk-uuid-1234' } } } }
+      expected_env = { 'disks' => { 'persistent' => { 'disk-uuid-1234' => nil } } }
+      expected_mountpoint = File.join(@disk_root, 'bind_mount_points', 'vm-uuid-1234', 'disk-uuid-1234')
+      expect(@disk_util).to receive(:disk_exist?).with('disk-uuid-1234').and_return(true)
+      expect(@disk_util).to receive(:umount_disk).with(expected_mountpoint)
+      expect(@cloud).to receive(:set_agent_env).with('vm-uuid-1234', expected_env)
+      expect(@cloud).to receive(:has_vm?).with('vm-uuid-1234').and_return(true)
+      @cloud.detach_disk('vm-uuid-1234', 'disk-uuid-1234')
+    end
+  end
+
+end