bosh_warden_cpi 1.5.0.pre.3 → 1.2513.0

data/README

@@ -2,15 +2,7 @@
 
 # Running with Vagrant up
 
-First things first: if you find that the "chef/cookbooks" directory is not present, do
+Now we use bosh-lite to bootstrap the local vm with warden cpi enabled:
+here is howto: 
 
-  cd chef
-  librarian-chef install
-
-
-Later on, after the vagrant box is set up and the director + Warden is successfully running,
-you'll probably have to download a stemcell from s3 and then upload it to the director.
-
-we need a warden-friendly stemcell that includes the director. micro_bosh stemcell did not work. Let's try vsphere next
-
-No worries, we'll soon work this into the vagrantfile
+https://github.com/cloudfoundry/bosh-lite/blob/master/README.md

data/lib/cloud/warden/cloud.rb

@@ -1,17 +1,13 @@
 module Bosh::WardenCloud
   class Cloud < Bosh::Cloud
 
-    include Helpers
+    include Bosh::WardenCloud::Helpers
 
-    DEFAULT_WARDEN_SOCK = "/tmp/warden.sock"
-    DEFAULT_STEMCELL_ROOT = "/var/vcap/stemcell"
-    DEFAULT_DISK_ROOT = "/var/vcap/store/disk"
-    DEFAULT_FS_TYPE = "ext4"
-    DEFAULT_POOL_SIZE = 128
-    DEFAULT_POOL_START_NUMBER = 10
-    DEFAULT_DEVICE_PREFIX = "/dev/sd"
-
-    DEFAULT_SETTINGS_FILE = "/var/vcap/bosh/settings.json"
+    DEFAULT_STEMCELL_ROOT = '/var/vcap/stemcell'
+    DEFAULT_DISK_ROOT = '/var/vcap/store/disk'
+    DEFAULT_FS_TYPE = 'ext4'
+    DEFAULT_WARDEN_DEV_ROOT = '/warden-cpi-dev'
+    DEFAULT_WARDEN_SOCK = '/tmp/warden.sock'
 
     attr_accessor :logger
 
@@ -22,47 +18,28 @@ module Bosh::WardenCloud
     def initialize(options)
       @logger = Bosh::Clouds::Config.logger
 
-      @agent_properties = options["agent"] || {}
-      @warden_properties = options["warden"] || {}
-      @stemcell_properties = options["stemcell"] || {}
-      @disk_properties = options["disk"] || {}
-
-      setup_warden
-      setup_stemcell
-      setup_disk
+      @agent_properties = options.fetch('agent', {})
+      @warden_properties = options.fetch('warden', {})
+      @stemcell_properties = options.fetch('stemcell', {})
+      @disk_properties = options.fetch('disk', {})
 
-      setup_pool
+      setup_path
+      @disk_utils = DiskUtils.new(@disk_root, @stemcell_root, @fs_type)
     end
 
     ##
     # Create a stemcell using stemcell image
-    # This method simply untar the stemcell image to a local directory. Warden
-    # can use the rootfs within the image as a base fs.
     # @param [String] image_path local path to a stemcell image
     # @param [Hash] cloud_properties not used
     # return [String] stemcell id
     def create_stemcell(image_path, cloud_properties)
       not_used(cloud_properties)
-
-      stemcell_id = uuid("stemcell")
-      stemcell_dir = stemcell_path(stemcell_id)
-
+      stemcell_id = uuid('stemcell')
       with_thread_name("create_stemcell(#{image_path}, _)") do
-
-        # Extract to tarball
-        @logger.info("Extracting stemcell from #{image_path} to #{stemcell_dir}")
-        FileUtils.mkdir_p(stemcell_dir)
-
-        # This command needs priviledge because the stemcell contains device files,
-        # which cannot be untared without priviledge
-        sudo "tar -C #{stemcell_dir} -xzf #{image_path} 2>&1"
-
+        @logger.info("Extracting stemcell from #{image_path} for #{stemcell_id}")
+        @disk_utils.stemcell_unpack(image_path, stemcell_id)
         stemcell_id
       end
-    rescue => e
-      sudo "rm -rf #{stemcell_dir}"
-
-      raise e
     end
 
     ##
@@ -71,9 +48,7 @@ module Bosh::WardenCloud
     # @return [void]
     def delete_stemcell(stemcell_id)
       with_thread_name("delete_stemcell(#{stemcell_id}, _)") do
-        stemcell_dir = stemcell_path(stemcell_id)
-        sudo "rm -rf #{stemcell_dir}"
-
+        @disk_utils.stemcell_delete(stemcell_id)
         nil
       end
     end
@@ -91,79 +66,40 @@ module Bosh::WardenCloud
     # @param [optional, Hash] env environment that will be passed to this vm
     # @return [String] vm_id
     def create_vm(agent_id, stemcell_id, resource_pool,
-                  networks, disk_locality = nil, env = nil)
+                  networks, disk_locality = nil, environment = nil)
       not_used(resource_pool)
       not_used(disk_locality)
-      not_used(env)
-
-      vm = nil
 
+      vm_handle = nil
       with_thread_name("create_vm(#{agent_id}, #{stemcell_id}, #{networks})") do
+        stemcell_path = @disk_utils.stemcell_path(stemcell_id)
+        vm_id = uuid('vm')
 
-        stemcell_path = stemcell_path(stemcell_id)
-
-        if networks.size > 1
-          raise ArgumentError, "Not support more than 1 nics"
-        end
-
-        unless Dir.exist?(stemcell_path)
-          cloud_error("Cannot find Stemcell(#{stemcell_id})")
-        end
-
-        vm = Models::VM.create
+        raise ArgumentError, 'Not support more than 1 nics' if networks.size > 1
+        cloud_error("Cannot find Stemcell(#{stemcell_id})") unless Dir.exist?(stemcell_path)
 
         # Create Container
-        handle = with_warden do |client|
+        vm_handle = with_warden do |client|
           request = Warden::Protocol::CreateRequest.new
+          request.handle = vm_id
           request.rootfs = stemcell_path
-          if networks.first[1]["type"] != "dynamic"
-            request.network = networks.first[1]["ip"]
+          if networks.first[1]['type'] != 'dynamic'
+            request.network = networks.first[1]['ip']
           end
-
+          request.bind_mounts = bind_mount_prepare(vm_id)
           response = client.call(request)
           response.handle
         end
-        vm.container_id = handle
+        cloud_error("Cannot create vm with given handle #{vm_id}") unless vm_handle == vm_id
 
         # Agent settings
-        env = generate_agent_env(vm, agent_id, networks)
-        set_agent_env(vm.container_id, env)
-
-        # Notice: It's a little hacky, but it's the way it is now.
-        #
-        # Warden has a default white list for devices. By default, all the loop
-        # devices cannot be read/written/mknod. We don't want to change the
-        # warden behavior, so we just manipulate the container cgroup directly.
-        sudo "bash -c 'echo \"b 7:* rwm\" > /tmp/warden/cgroup/devices/instance-#{handle}/devices.allow'"
-
-        # Start bosh agent
-        with_warden do |client|
-          request = Warden::Protocol::SpawnRequest.new
-          request.handle = handle
-          request.privileged = true
-          request.script = "/usr/sbin/runsvdir-start"
-
-          client.call(request)
-        end
-
-        # Save to DB
-        vm.save
-
-        vm.id.to_s
+        env = generate_agent_env(vm_id, agent_id, networks, environment)
+        set_agent_env(vm_id, env)
+        start_agent(vm_id)
+        vm_id
       end
     rescue => e
-      if vm
-        if vm.container_id
-          with_warden do |client|
-            request = Warden::Protocol::DestroyRequest.new
-            request.handle = vm.container_id
-
-            client.call(request)
-          end
-        end
-
-        vm.destroy
-      end
+      destroy_container(vm_handle) if vm_handle
       raise e
     end
 
@@ -174,25 +110,39 @@ module Bosh::WardenCloud
     # @return [void]
     def delete_vm(vm_id)
       with_thread_name("delete_vm(#{vm_id})") do
-        vm = Models::VM[vm_id.to_i]
-
-        cloud_error("Cannot find VM #{vm}") unless vm
-        cloud_error("Cannot delete vm with disks attached") if vm.disks.size > 0
-
-        container_id = vm.container_id
-
-        vm.destroy
-
-        with_warden do |client|
-          request = Warden::Protocol::DestroyRequest.new
-          request.handle = container_id
-
-          client.call(request)
+        if has_vm?(vm_id)
+          destroy_container(vm_id)
+          vm_bind_mount = File.join(@bind_mount_points, vm_id)
+          sudo "umount #{vm_bind_mount}"
         end
 
+        ephemeral_mount = File.join(@ephemeral_mount_points, vm_id)
+        sudo "rm -rf #{ephemeral_mount}"
+        vm_bind_mount = File.join(@bind_mount_points, vm_id)
+        sudo "rm -rf #{vm_bind_mount}"
         nil
       end
+    end
+
+    ##
+    # Checks if a VM exists
+    #
+    # @param [String] vm_id vm id
+    # @return [Boolean] True if the vm exists
 
+    def has_vm?(vm_id)
+      with_thread_name("has_vm(#{vm_id})") do
+        result = false
+        handles = with_warden do |client|
+          request = Warden::Protocol::ListRequest.new
+          response = client.call(request)
+          response.handles
+        end
+        unless handles.nil?
+          result = handles.include?(vm_id)
+        end
+        result
+      end
     end
 
     def reboot_vm(vm_id)
@@ -212,45 +162,11 @@ module Bosh::WardenCloud
     # @return [String] disk id
     def create_disk(size, vm_locality = nil)
       not_used(vm_locality)
-
-      disk = nil
-      number = nil
-      image_file = nil
-
-      raise ArgumentError, "disk size <= 0" unless size > 0
-
       with_thread_name("create_disk(#{size}, _)") do
-        disk = Models::Disk.create
-
-        image_file = image_path(disk.id)
-
-        FileUtils.touch(image_file)
-        File.truncate(image_file, size << 20) # 1 MB == 1<<20 Byte
-        sh "mkfs -t #{@fs_type} -F #{image_file} 2>&1"
-
-        # Get a device number from the pool
-        number = @pool.acquire
-        cloud_error("Failed to fetch device number") unless number
-
-        # Attach image file to the device
-        sudo "losetup /dev/loop#{number} #{image_file}"
-
-        disk.image_path = image_file
-        disk.device_num = number
-        disk.attached = false
-        disk.save
-
-        disk.id.to_s
+        disk_id = uuid('disk')
+        @disk_utils.create_disk(disk_id, size)
+        disk_id
       end
-    rescue => e
-      if number
-        sudo "losetup -d /dev/loop#{number}"
-        @pool.release(number)
-      end
-      FileUtils.rm_f image_file if image_file
-      disk.destroy if disk
-
-      raise e
     end
 
     ##
@@ -260,23 +176,8 @@ module Bosh::WardenCloud
     # @return [void]
     def delete_disk(disk_id)
       with_thread_name("delete_disk(#{disk_id})") do
-        disk = Models::Disk[disk_id.to_i]
-
-        cloud_error("Cannot find disk #{disk_id}") unless disk
-        cloud_error("Cannot delete attached disk") if disk.attached
-
-        # Detach image file from loop device
-        sudo "losetup -d /dev/loop#{disk.device_num}"
-
-        # Release the device number back to pool
-        @pool.release(disk.device_num)
-
-        # Delete DB entry
-        disk.destroy
-
-        # Remove image file
-        FileUtils.rm_f image_path(disk_id)
-
+        cloud_error("Cannot find disk #{disk_id}") unless has_disk?(disk_id)
+        @disk_utils.delete_disk(disk_id)
         nil
       end
     end
@@ -289,38 +190,17 @@ module Bosh::WardenCloud
     # @return nil
     def attach_disk(vm_id, disk_id)
       with_thread_name("attach_disk(#{vm_id}, #{disk_id})") do
-        vm = Models::VM[vm_id.to_i]
-        disk = Models::Disk[disk_id.to_i]
-
-        cloud_error("Cannot find vm #{vm_id}") unless vm
-        cloud_error("Cannot find disk #{disk_id}") unless disk
-        cloud_error("Disk #{disk_id} already attached") if disk.attached
-
-        # Create a device file inside warden container
-        script = attach_script(disk.device_num, @device_path_prefix)
+        cloud_error("Cannot find vm #{vm_id}") unless has_vm?(vm_id)
+        cloud_error("Cannot find disk #{disk_id}") unless has_disk?(disk_id)
 
-        device_path = with_warden do |client|
-          request = Warden::Protocol::RunRequest.new
-          request.handle = vm.container_id
-          request.script = script
-          request.privileged = true
-
-          response = client.call(request)
-
-          stdout = response.stdout || ""
-          stdout.strip
-        end
+        vm_bind_mount = File.join(@bind_mount_points, vm_id)
+        disk_dir = File.join(vm_bind_mount, disk_id)
 
+        @disk_utils.mount_disk(disk_dir, disk_id)
         # Save device path into agent env settings
-        env = get_agent_env(vm.container_id)
-        env["disks"]["persistent"][disk_id] = device_path
-        set_agent_env(vm.container_id, env)
-
-        # Save DB entry
-        disk.device_path = device_path
-        disk.attached = true
-        disk.vm = vm
-        disk.save
+        env = get_agent_env(vm_id)
+        env['disks']['persistent'][disk_id] = File.join(@warden_dev_root, disk_id)
+        set_agent_env(vm_id, env)
 
         nil
       end
@@ -334,38 +214,19 @@ module Bosh::WardenCloud
     # @return nil
     def detach_disk(vm_id, disk_id)
       with_thread_name("detach_disk(#{vm_id}, #{disk_id})") do
-        vm = Models::VM[vm_id.to_i]
-        disk = Models::Disk[disk_id.to_i]
 
-        cloud_error("Cannot find vm #{vm_id}") unless vm
-        cloud_error("Cannot find disk #{disk_id}") unless disk
-        cloud_error("Disk #{disk_id} not attached") unless disk.attached
+        cloud_error("Cannot find vm #{vm_id}") unless has_vm?(vm_id)
+        cloud_error("Cannot find disk #{disk_id}") unless has_disk?(disk_id)
 
-        device_num = disk.device_num
-        device_path = disk.device_path
+        vm_bind_mount = File.join(@bind_mount_points, vm_id)
+        device_path = File.join(vm_bind_mount, disk_id)
 
+        # umount the image file
+        @disk_utils.umount_disk(device_path)
         # Save device path into agent env settings
-        env = get_agent_env(vm.container_id)
-        env["disks"]["persistent"][disk_id] = nil
-        set_agent_env(vm.container_id, env)
-
-        # Save DB entry
-        disk.attached = false
-        disk.device_path = nil
-        disk.vm = nil
-        disk.save
-
-        # Remove the device file and partition file inside warden container
-        script = "rm #{partition_path(device_path)} #{device_path}"
-
-        with_warden do |client|
-          request = Warden::Protocol::RunRequest.new
-          request.handle = vm.container_id
-          request.script = script
-          request.privileged = true
-
-          client.call(request)
-        end
+        env = get_agent_env(vm_id)
+        env['disks']['persistent'][disk_id] = nil
+        set_agent_env(vm_id, env)
 
         nil
       end
@@ -377,124 +238,56 @@ module Bosh::WardenCloud
 
     private
 
-    def not_used(*arg)
-      # no-op
-    end
-
-    def stemcell_path(stemcell_id)
-      File.join(@stemcell_root, stemcell_id)
-    end
-
-    def image_path(disk_id)
-      File.join(@disk_root, "#{disk_id}.img")
+    def has_disk?(disk_id)
+      @disk_utils.disk_exist?(disk_id)
     end
 
-    def setup_warden
-      @warden_unix_path = @warden_properties["unix_domain_path"] || DEFAULT_WARDEN_SOCK
-    end
-
-    def setup_stemcell
-      @stemcell_root = @stemcell_properties["root"] || DEFAULT_STEMCELL_ROOT
-
-      FileUtils.mkdir_p(@stemcell_root)
-    end
-
-    def setup_disk
-      @disk_root = @disk_properties["root"] || DEFAULT_DISK_ROOT
-      @fs_type = @disk_properties["fs"] || DEFAULT_FS_TYPE
-      @pool_size = @disk_properties["pool_count"] || DEFAULT_POOL_SIZE
-      @pool_start_number = @disk_properties["pool_start_number"] || DEFAULT_POOL_START_NUMBER
-      @device_path_prefix = @disk_properties["device_path_prefix"] || DEFAULT_DEVICE_PREFIX
-
-      FileUtils.mkdir_p(@disk_root)
-    end
-
-    def setup_pool
-      @pool = DevicePool.new(@pool_size) { |i| i + @pool_start_number }
-
-      occupied_numbers = Models::Disk.collect { |disk| disk.device_num }
-      @pool.delete_if do |i|
-        occupied_numbers.include? i
-      end
-
-      # Initialize the loop devices
-      last = @pool_start_number + @pool_size - 1
-      @pool_start_number.upto(last) do |i|
-        sudo "mknod /dev/loop#{i} b 7 #{i}" unless File.exists? "/dev/loop#{i}"
-      end
+    def not_used(*arg)
+      # no-op
     end
 
-    def with_warden
-      client = Warden::Client.new(@warden_unix_path)
-      client.connect
+    def setup_path
+      @warden_unix_path = @warden_properties.fetch('unix_domain_path', DEFAULT_WARDEN_SOCK)
+      @warden_dev_root = @disk_properties.fetch('warden_dev_root', DEFAULT_WARDEN_DEV_ROOT)
+      @stemcell_root = @stemcell_properties.fetch('root', DEFAULT_STEMCELL_ROOT)
 
-      ret = yield client
+      @disk_root = @disk_properties.fetch('root', DEFAULT_DISK_ROOT)
+      @fs_type = @disk_properties.fetch('fs', DEFAULT_FS_TYPE)
 
-      ret
-    ensure
-      client.disconnect if client
+      @bind_mount_points = File.join(@disk_root, 'bind_mount_points')
+      @ephemeral_mount_points = File.join(@disk_root, 'ephemeral_mount_point')
     end
 
-    def agent_settings_file
-      DEFAULT_SETTINGS_FILE
-    end
+    def bind_mount_prepare(vm_id)
+      vm_bind_mount = File.join(@bind_mount_points, vm_id)
+      FileUtils.mkdir_p(vm_bind_mount)
+      vm_ephemeral_mount = File.join(@ephemeral_mount_points, vm_id)
+      FileUtils.mkdir_p(vm_ephemeral_mount)
 
-    def generate_agent_env(vm, agent_id, networks)
-      vm_env = {
-        "name" => vm.container_id,
-        "id" => vm.id
-      }
-
-      env = {
-        "vm" => vm_env,
-        "agent_id" => agent_id,
-        "networks" => networks,
-        "disks" => { "persistent" => {} },
-      }
-      env.merge!(@agent_properties)
-      env
-    end
+      # Make the bind mount point shareable
+      sudo "mount --bind #{vm_bind_mount} #{vm_bind_mount}"
+      sudo "mount --make-unbindable #{vm_bind_mount}"
+      sudo "mount --make-shared #{vm_bind_mount}"
 
-    def get_agent_env(handle)
-      body = with_warden do |client|
-        request = Warden::Protocol::RunRequest.new
-        request.handle = handle
-        request.privileged = true
-        request.script = "cat #{agent_settings_file}"
+      bind_mount = Warden::Protocol::CreateRequest::BindMount.new
+      bind_mount.src_path = vm_bind_mount
+      bind_mount.dst_path = @warden_dev_root
+      bind_mount.mode = Warden::Protocol::CreateRequest::BindMount::Mode::RW
 
-        client.call(request).stdout
-      end
+      ephemeral_mount = Warden::Protocol::CreateRequest::BindMount.new
+      ephemeral_mount.src_path = vm_ephemeral_mount
+      ephemeral_mount.dst_path = '/var/vcap/data'
+      ephemeral_mount.mode = Warden::Protocol::CreateRequest::BindMount::Mode::RW
 
-      env = Yajl::Parser.parse(body)
-      env
+      return [bind_mount, ephemeral_mount]
     end
 
-    def set_agent_env(handle, env)
-      tempfile = Tempfile.new("settings")
-      tempfile.write(Yajl::Encoder.encode(env))
-      tempfile.close
-
-      tempfile_in = "/tmp/#{rand(100000)}"
-
-      # Here we copy the setting file to temp file in container, then mv it to
-      # /var/vcap/bosh by privileged user.
+    def destroy_container(container_id)
       with_warden do |client|
-        request = Warden::Protocol::CopyInRequest.new
-        request.handle = handle
-        request.src_path = tempfile.path
-        request.dst_path = tempfile_in
-
-        client.call(request)
-
-        request = Warden::Protocol::RunRequest.new
-        request.handle = handle
-        request.privileged = true
-        request.script = "mv #{tempfile_in} #{agent_settings_file}"
-
+        request = Warden::Protocol::DestroyRequest.new
+        request.handle = container_id
         client.call(request)
       end
-
-      tempfile.unlink
     end
 
   end