deltacloud-core 0.3.0 → 0.4.0

data/lib/deltacloud/drivers/vsphere/vsphere_filemanager.rb

@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.  The
+# ASF licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the
+# License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+require 'nokogiri'
+
+module VSphere
+  module FileManager
+
+
+  DIRECTORY_PATH="deltacloud"
+  MKISOFS_EXECUTABLE="mkisofs"
+  # This value is setted in this way because
+  # mkisofs man said, less than this amount
+  # he have to pad the content of the iso file
+  # that mean a limit of 400 kb file since
+  # 1 sector of iso file = 2048 bytes
+  ISO_SECTORS=200
+
+  RbVmomi::VIM::Datastore::class_eval do
+    def soap
+      @soap
+    end
+  end
+
+  class << self
+
+    def store_iso!(datastore,base64_iso, file_name)
+      file = StringIO.new(get_plain_iso(base64_iso).read)
+      uploadFile(datastore, file, file_name)
+    end
+
+    def user_data!(datastore,base64_content,file_name)
+      command="#{MKISOFS_EXECUTABLE} -stream-file-name #{file_name}.txt -stream-media-size #{ISO_SECTORS}"
+      iso_file=''
+      Open3::popen3(command) do |stdin, stdout, stderr|
+        stdin.write(base64_content.unpack("m"))
+        stdin.close()
+        iso_file=StringIO::new(stdout.read)
+      end
+      uploadFile(datastore,iso_file,file_name)
+    end
+
+
+    def delete_iso!(datastore,file_name)
+      deleteFile(datastore, file_name)
+    end
+
+    def store_mapping!(datastore, yaml_object, file_name)
+      file = StringIO::new(yaml_object)
+      uploadFile(datastore, file, file_name)
+    end
+
+    def delete_mapping!(datastore, file_name)
+      deleteFile(datastore, file_name)
+    end
+
+    def load_mapping(datastore, file_name)
+      YAML::load(downloadFile(datastore, file_name))
+    end
+
+    def list_mappings(datastore)
+      listFolder(datastore)
+    end
+
+   private
+
+    def make_directory!(datastore,directory)
+      dc=datastore.send(:datacenter)
+      dc._connection.serviceContent.fileManager.MakeDirectory :name => "[#{datastore.name}] #{directory}",
+                                                              :datacenter => dc,
+                                                              :createParentDirectories => false
+    end
+
+    def _exist?(datastore,file=nil)
+      uri = buildUrl(datastore,file) if file
+      uri = buildUrl(datastore) unless file
+      http = Net::HTTP.new(uri.host,uri.port)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      headers = {
+        'cookie' => datastore.send(:soap).cookie,
+      }
+      request = Net::HTTP::Head.new(uri.request_uri,headers)
+      res = http.request(request)
+      if res.kind_of?(Net::HTTPSuccess)
+        return true
+      else
+        return false
+      end
+    end
+
+    def downloadFile(datastore,file_name)
+      @uri = buildUrl(datastore,file_name)
+      http=Net::HTTP.new(@uri.host, @uri.port)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      headers = { 'cookie' => datastore.send(:soap).cookie }
+      request = Net::HTTP::Get.new(@uri.request_uri, headers)
+      res = http.request(request)
+      raise "download failed: #{res.message}" unless res.kind_of?(Net::HTTPSuccess)
+      res.body
+    end
+
+    def deleteFile(datastore, file)
+      @uri = buildUrl(datastore, file)
+      http=Net::HTTP.new(@uri.host, @uri.port)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      headers = { 'cookie' => datastore.send(:soap).cookie }
+      request = Net::HTTP::Delete.new(@uri.request_uri, headers)
+      res = http.request(request)
+      unless res.kind_of?(Net::HTTPSuccess) or res.kind_of?(Net::HTTPServiceUnavailable) or res.kind_of?(Net::HTTPNotFound)
+        raise "delete failed: #{res.message} #{file}"
+      end
+    end
+
+    def listFolder(datastore, folder="")
+      @uri = buildUrl(datastore, folder)
+      http=Net::HTTP.new(@uri.host, @uri.port)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      headers = { 'cookie' => datastore.send(:soap).cookie }
+      request = Net::HTTP::Get.new(@uri.request_uri, headers)
+      begin
+        res = http.request(request)
+        Nokogiri::HTML(res.body).css("table tr a").map { |f| f.text.strip }.reject { |f| f == 'Parent Directory'}
+      rescue
+        puts "[ERROR]: Unable to list deltacloud folder"
+        []
+      end
+    end
+
+    def uploadFile(datastore,file,file_name)
+      raise "You need to set the realm_id" if datastore.nil?
+      make_directory!(datastore,DIRECTORY_PATH) unless _exist?(datastore)
+      @uri = buildUrl(datastore,file_name)
+      http=Net::HTTP.new(@uri.host, @uri.port)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      headers = {
+        'cookie' => datastore.send(:soap).cookie,
+        'content-length' => file.size.to_s,
+        'Content-Type' => 'application/octet-stream',
+      }
+      request = Net::HTTP::Put.new(@uri.request_uri, headers)
+      request.body_stream = file
+      res = http.request(request)
+      raise "upload failed: #{res.message}" unless res.kind_of?(Net::HTTPSuccess)
+    end
+
+    # return the url like https://<server_address>/folder/<path>/<file_name>?<query_infos>
+
+    def buildUrl(datastore,file="")
+      uri=URI::HTTPS::build(:host=>datastore._connection.http.address)
+      uri.path= ["/folder",DIRECTORY_PATH,file].join("/") if file
+      query={:dcPath => datastore.send(:datacenter).name, :dsName => datastore.name }
+      uri.query=query.collect{|name, value| "#{name}=#{URI.escape value}"}.join("&")
+      uri
+    end
+
+    def get_plain_iso(stream)
+      unbase64file=stream.unpack('m').to_s
+      Zlib::GzipReader.new(StringIO.new(unbase64file))
+    end
+
+   end
+
+  end
+end

data/lib/deltacloud/hardware_profile.rb

@@ -136,7 +136,7 @@ module Deltacloud
       props = []
       self.each_property do |p|
         if p.kind.eql? :fixed
-          props << { :kind => p.kind, :value => p.value, :name => p.name, :unit => p.unit } 
+          props << { :kind => p.kind, :value => p.value, :name => p.name, :unit => p.unit }
         else
           param = { :operation => "create", :method => "post", :name => p.name }
           if p.kind.eql? :range

data/lib/deltacloud/helpers.rb

@@ -15,7 +15,8 @@
 # under the License.
 
 require 'deltacloud/helpers/application_helper'
+require 'deltacloud/helpers/json_helper'
 require 'deltacloud/helpers/conversion_helper'
 require 'deltacloud/helpers/hardware_profiles_helper'
 
-helpers ApplicationHelper, ConversionHelper, HardwareProfilesHelper
+helpers ApplicationHelper, ConversionHelper, HardwareProfilesHelper, JSONHelper

data/lib/deltacloud/helpers/application_helper.rb

@@ -22,17 +22,17 @@ module ApplicationHelper
   include Deltacloud
 
   def bread_crumb
-    s = "<ul class='breadcrumb'><li class='first'><a href='#{url_for('/')}'>&#948</a></li>"
+    s = "<ul class='breadcrumb'><li class='first'><a href='#{root_url}'>&#948</a></li>"
     url = request.path_info.split('?')  #remove extra query string parameters
     levels = url[0].split('/') #break up url into different levels
     levels.each_with_index do |level, index|
       unless level.blank?
-        if index == levels.size-1 ||
-           (level == levels[levels.size-2] && levels[levels.size-1].to_i > 0)
+        next if "/#{level}" == Sinatra::UrlForHelper::DEFAULT_URI_PREFIX
+        if index == levels.size-1 || (level == levels[levels.size-2] && levels[levels.size-1].to_i > 0)
           s += "<li class='subsequent'>#{level.gsub(/_/, ' ')}</li>\n" unless level.to_i > 0
         else
-            link = levels.slice(0, index+1).join("/")
-            s += "<li class='subsequent'><a href=\"#{url_for(link)}\">#{level.gsub(/_/, ' ')}</a></li>\n"
+          link = levels.slice(2, index-1).join("/")
+          s += "<li class='subsequent'><a href=\"#{api_url_for(link)}\">#{level.gsub(/_/, ' ')}</a></li>\n"
         end
       end
     end
@@ -100,16 +100,18 @@ module ApplicationHelper
         format.json { convert_to_json(model, @element) }
       end
     else
-        report_error(404, 'not_found')
+        report_error(404)
     end
   end
 
-  def report_error(status, template)
-    @error = request.env['sinatra.error']
-    response.status = status
+  def report_error(code=nil)
+    @error, @code = request.env['sinatra.error'], code
+    @code = 500 if not @code and not @error.class.method_defined? :code
+    response.status = @code || @error.code
     respond_to do |format|
-      format.xml { haml :"errors/#{template}", :layout => false }
-      format.html { haml :"errors/#{template}", :layout => :error }
+      format.xml {  haml :"errors/#{@code || @error.code}", :layout => false }
+      format.json { json_return_error(@error) }
+      format.html { haml :"errors/#{@code || @error.code}", :layout => :error }
     end
   end
 
@@ -119,7 +121,7 @@ module ApplicationHelper
     # If original instance doesn't include called action
     # return with 405 error (Method is not Allowed)
     unless driver.instance_actions_for(original_instance.state).include?(name.to_sym)
-      return report_error(405, 'not_allowed')
+      return report_error(405)
     end
 
     @instance = driver.send(:"#{name}_instance", credentials, params["id"])
@@ -145,14 +147,14 @@ module ApplicationHelper
   end
 
   def render_cdata(text)
-    "<pem><![CDATA[#{text.strip}]]></pem>"
+    "<![CDATA[#{text.strip}]]>"
   end
 
   def link_to_action(action, url, method)
     capture_haml do
       haml_tag :form, :method => :post, :action => url, :class => [:link, method] do
         haml_tag :input, :type => :hidden, :name => '_method', :value => method
-        haml_tag :button, :type => :submit do 
+        haml_tag :button, :type => :submit, :'data-ajax' => 'false', :'data-inline' => "true" do
           haml_concat action
         end
       end
@@ -168,25 +170,26 @@ module ApplicationHelper
     else
       uri+="?format=#{format}"
     end
-    '<a href="%s">%s</a>' % [uri, "#{format}".upcase]
+    '<a data-ajax="false" data-icon="grid" href="%s">%s</a>' % [uri, "#{format}".upcase]
   end
 
   def link_to_documentation
     return '' unless request.env['REQUEST_URI']
     uri = request.env['REQUEST_URI'].dup
-    uri.gsub!('/api', '/api/docs/') unless uri.include?("docs") #i.e. if already serving under /api/docs, leave it be
+    uri.gsub!(Sinatra::UrlForHelper::DEFAULT_URI_PREFIX,
+              api_url_for(:docs)) unless uri.include?("docs") #i.e. if already serving under /api/docs, leave it be
     '<a href="%s">[ Documentation ]</a>' % uri
   end
 
   def action_url
     if [:index].include?(@operation.name)
-      url_for("/api/#{@collection.name.to_s}")
+      api_url_for("#{@collection.name.to_s}")
     elsif [:show, :stop, :start, :reboot, :attach, :detach].include?(@operation.name)
-      url_for("/api/#{@collection.name.to_s}/:id/#{@operation.name}")
+      api_url_for("#{@collection.name.to_s}/:id/#{@operation.name}")
     elsif [:destroy].include?(@operation.name)
-      url_for("/api/#{@collection.name.to_s}/:id")
+      api_url_for("#{@collection.name.to_s}/:id")
     else
-      url_for("/api/#{@collection.name}/#{@operation.name}")
+      api_url_for("#{@collection.name}/#{@operation.name}")
     end
   end
 
@@ -216,4 +219,73 @@ module ApplicationHelper
     end
     result
   end
+
+  def header(title, opts={}, &block)
+    opts[:theme] ||= 'b'
+    opts[:back] ||= 'true'
+    capture_haml do
+      haml_tag :div, :'data-role' => :header, :'data-theme' => opts[:theme], :'data-add-back-btn' => opts[:back] do
+        haml_tag :a, :'data-rel' => :back do
+          haml_concat "Back"
+        end if opts[:back] == 'true'
+        haml_tag :h1 do
+          haml_concat title
+        end
+        block.call if block_given?
+      end
+    end
+  end
+
+  def subheader(title, opts={})
+    opts[:theme] ||= 'a'
+    capture_haml do
+      haml_tag :div, :'data-role' => :header, :'data-theme' => opts[:theme] do
+        haml_tag :p, :class => 'inner-right' do
+          haml_concat title
+        end
+      end
+    end
+  end
+
+  # FIXME: It would be cleaner if we stored the type of address explicitly in
+  # public_addresses instead of guessing it; especially since now a RHEV-M
+  # vnc address in theory could look like type ipv4.
+  #
+  # Instead of pushing just the address onto public_addresses, we should
+  # just push a pair [type, address], i.e. [:vnc, "172.16.0.1"] or a hash
+  # { :vnc => "172.16.0.1" }
+  #
+  def address_type(address)
+    case address
+      when /^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})?$/; :ipv4
+      when /^.*:([\-\d]+)$/; :vnc
+      when /^(\S{1,2}:\S{1,2}:\S{1,2}:\S{1,2}:\S{1,2}:\S{1,2})?$/; :mac
+      else :hostname
+    end
+  end
+
+  def format_address(address, opts={})
+    capture_haml do
+      haml_tag :address, { :type => address_type(address) }.merge(opts) do
+        haml_concat address
+      end
+    end
+  end
+
+  def translate_error_code(code)
+    case code
+      when 400; { :message => "Bad Request" }
+      when 401; { :message => "Unauthorized" }
+      when 403; { :message => "Forbidden" }
+      when 404; { :message => "Not Found" }
+      when 405; { :message => "Method Not Allowed" }
+      when 406; { :message => "Not Acceptable" }
+      when 500; { :message => "Internal Server Error" }
+      when 502; { :message => "Backend Server Error" }
+    end
+  end
+
+  def new_blob_form_url(bucket)
+    bucket_url(@bucket.name) + "/" + NEW_BLOB_FORM_ID
+  end
 end

data/lib/deltacloud/helpers/blob_stream.rb

@@ -17,7 +17,8 @@ include Deltacloud
 begin
   require 'eventmachine'
   #--
-  # based on the example from http://macournoyer.com/blog/2009/06/04/pusher-and-async-with-thin/
+  # based on the example from
+  #   http://macournoyer.com/blog/2009/06/04/pusher-and-async-with-thin/
   #--
   class BlobStream
     AsyncResponse = [-1, {}, []].freeze
@@ -30,13 +31,13 @@ begin
       EM.next_tick { env['async.callback'].call [200, {
         'Content-Type' => "#{params['content_type']}",
         'Content-Disposition' => params["content_disposition"],
-        'Content-Length' => "#{params['content_length']}"}, body] 
+        'Content-Length' => "#{params['content_length']}"}, body]
       }
-      #call the driver from here. the driver method yields for every chunk of blob it receives. We then
-      #use body.call to write that chunk as received.
+      #call the driver from here. the driver method yields for every chunk
+      #of blob it receives. Then use body.call to write that chunk as received.
       driver.blob_data(credentials, params[:bucket], params[:blob], params) {|chunk| body.call ["#{chunk}"]} #close blob_data block
       body.succeed
-      AsyncResponse # Tells Thin to not close the connection and continue it's work on other request
+      AsyncResponse # Tell Thin to not close connection & work other requests
     end
   end
 
@@ -64,18 +65,165 @@ end
 
 class Hash
 
-  def gsub_keys(pattern, replacement)
-    rgx_pattern = Regexp.compile(pattern, true)
+  def gsub_keys(rgx_pattern, replacement)
     remove = []
     self.each_key do |key|
       if key.to_s.match(rgx_pattern)
-         new_key = key.to_s.gsub(rgx_pattern, replacement)
+         new_key = key.to_s.gsub(rgx_pattern, replacement).downcase
          self[new_key] = self[key]
          remove << key
-      end #key.match
-    end # each_key do
+      end
+    end
     #remove the original keys
     self.delete_if{|k,v| remove.include?(k)}
-  end #def
+  end
+
+end
+
+module BlobHelper
+
+  def self.extract_blob_metadata_hash(env_hash)
+    meta_array = env_hash.select{|k,v| k.match(/^HTTP[-_]X[-_]Deltacloud[-_]Blobmeta[-_]/i)}
+    metadata = meta_array.inject({}){ |result, array| result[array.first.upcase] = array.last; result}
+    metadata
+  end
+
+DELTACLOUD_BLOBMETA_HEADER = /HTTP[-_]X[-_]Deltacloud[-_]Blobmeta[-_]/i
+
+  #e.g. from HTTP-X-Deltacloud-Blobmeta-FOO:BAR to amz-meta-FOO:BAR
+  def self.rename_metadata_headers(metadata, rename_to)
+    metadata.gsub_keys(DELTACLOUD_BLOBMETA_HEADER, rename_to)
+  end
+
+end
+
+#Monkey patch for streaming blobs:
+# Normally a client will upload a blob to deltacloud and thin will put
+# this into a tempfile. Then deltacloud would stream up to the provider:
+#   i.e.  client =-->>TEMP_FILE-->> deltacloud =-->>STREAM-->> provider
+# Instead we want to recognise that this is a 'PUT blob' operation and
+# start streaming to the provider as the request is received:
+#   i.e.  client =-->>STREAM-->> deltacloud =-->>STREAM-->> provider
+module Thin
+  class Request
+
+    alias_method :move_body_to_tempfile_orig, :move_body_to_tempfile if defined?(Thin::Response)
+    private
+      def move_body_to_tempfile
+        if BlobStreamIO::is_put_blob(self)
+          @body = BlobStreamIO.new(self)
+        else
+          move_body_to_tempfile_orig
+        end
+      end
+
+  end
+end
+
+require 'net/http'
+#monkey patch for Net:HTTP
+module Net
+  class HTTP
+
+    alias :request_orig :request
+
+    def request(req, body = nil, blob_stream = nil, &block)
+      unless blob_stream
+        return request_orig(req, body, &block)
+      end
+      @blob_req = req
+      do_start #start the connection
+
+      req.set_body_internal body
+      begin_transport req
+      req.write_header_m @socket,@curr_http_version, edit_path(req.path)
+      @socket
+    end
+
+    class Put < HTTPRequest
+      def write_header_m(sock, ver, path)
+        write_header(sock, ver, path)
+      end
+    end
+
+    def end_request
+      begin
+        res = HTTPResponse.read_new(@socket)
+      end while res.kind_of?(HTTPContinue)
+      res.reading_body(@socket, @blob_req.response_body_permitted?) {
+                                          yield res if block_given? }
+      end_transport @blob_req, res
+      do_finish
+      res
+    end
+  end
+
+end
+
+require 'base64'
+class BlobStreamIO
+
+  attr_accessor :size, :provider, :sock
+
+  def initialize(request)
+    @client_request = request
+    @size = 0
+    bucket, blob = parse_bucket_blob(request.env["PATH_INFO"])
+    user, password = parse_credentials(request.env['HTTP_AUTHORIZATION'])
+    content_type = request.env['CONTENT_TYPE'] || ""
+    #deal with blob_metadata: (X-Deltacloud-Blobmeta-name: value)
+    user_meta = BlobHelper::extract_blob_metadata_hash(request.env)
+    @content_length = request.env['CONTENT_LENGTH']
+    @http, provider_request = driver.blob_stream_connection({:user=>user,
+       :password=>password, :bucket=>bucket, :blob=>blob, :metadata=> user_meta,
+       :content_type=>content_type, :content_length=>@content_length })
+    @content_length = @content_length.to_i #for comparison of size in '<< (data)'
+    @sock = @http.request(provider_request, nil, true)
+  end
+
+  def << (data)
+    @sock.write(data)
+    @size += data.length
+    if (@size >= @content_length)
+      result = @http.end_request
+      if result.is_a?(Net::HTTPSuccess)
+        @client_request.env["BLOB_SUCCESS"] = "true"
+      else
+        @client_request.env["BLOB_FAIL"] = result.body
+      end
+    end
+  end
+
+  def rewind
+  end
+
+  #use the Request.env hash (populated by the ThinParser) to determine whether
+  #this is a post blob operation. By definition, only get here with a body of
+  # > 112kbytes - thin/lib/thin/request.rb:12 MAX_BODY = 1024 * (80 + 32)
+  def self.is_put_blob(request = nil)
+    path = request.env['PATH_INFO']
+    method = request.env['REQUEST_METHOD']
+    if ( path =~ /^#{Regexp.escape(Sinatra::UrlForHelper::DEFAULT_URI_PREFIX)}\/buckets/ && method == 'PUT' )
+      return true
+    else
+      return false
+    end
+  end
 
-end #class
+  private
+
+  def parse_bucket_blob(request_string)
+    array = request_string.split("/")
+    blob = array.pop
+    bucket = array.pop
+    return bucket, blob
+  end
+
+  def parse_credentials(request_string)
+    decoded = Base64.decode64(request_string.split('Basic ').last)
+    key = decoded.split(':').first
+    pass = decoded.split(':').last
+    return key, pass
+  end
+
+end