visor-image 0.0.1

data/lib/image/store/hdfs.rb

@@ -0,0 +1,149 @@
+require 'uri'
+require 'net/http'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+
+module Visor
+  module Image
+    module Store
+
+      # The Apache Hadoop HDFS (HDFS) backend store.
+      #
+      # This class handles the management of image files located in the HDFS storage system,
+      # based on a URI like *hdfs://username@s<host>:<port>/<path>/<bucket>/<image>*.
+      #
+      #http://10.0.3.12:50075/webhdfs/v1/foo/1.iso?op=OPEN&user.name=hadoop&offset=0
+      class HDFS
+        include Visor::Common::Exception
+
+        CONTEXT_ROOT="webhdfs/v1"
+
+        attr_accessor :uri, :config, :username, :bucket, :file, :base, :host, :port
+
+        # Initializes a new Cumulus store client object. Cumulus credentials are loaded from the URI,
+        # on GET and DELETE operations, or from the configuration file for POST and PUT operation.
+        #
+        # @param [String] uri The URI of the file location.
+        # @param config [Hash] A set of configurations for the wanted store, loaded from
+        #   VISoR configuration file.
+        #
+        # @return [Object] An instantiated Cumulus store object ready to use.
+        #
+        def initialize(uri, config)
+          @uri    = URI(uri)
+          @config = config[:hdfs]
+
+          if @uri.scheme
+            @username = @uri.user
+            @base     = @uri.path.split('/')[1..2].join('/')
+            @bucket   = @uri.path.split('/')[3]
+            @file     = @uri.path.split('/')[4]
+            @host     = @uri.host
+            @port     = @uri.port
+          else
+            @username = @config[:username]
+            @bucket   = @config[:bucket]
+            @base     = CONTEXT_ROOT
+            @host     = @config[:host]
+            @port     = @config[:port]
+          end
+        end
+
+
+        # Returns the image file to clients, streamed in chunks.
+        #
+        # @return [Object] Yields the file, a chunk at time.
+        #
+        def get
+          uri      = generate_uri('op=OPEN')
+          # This raises cant yield from root fiber
+          #res = EventMachine::HttpRequest.new(uri).get
+          #url = URI(res.response_header['LOCATION'])
+          #url.hostname = host
+          #http   = EventMachine::HttpRequest.new(url).aget
+          # ...
+
+          # This works, should substitute (uri).get with (url).get in down
+          #require "net/http"
+          #req          = Net::HTTP::Get.new(uri.request_uri)
+          #res          = Net::HTTP.new(uri.hostname, uri.port).request(req)
+          #url          = URI(res['location'])
+          #url.hostname = host
+          #STDERR.puts "URL #{url}"
+          # ...
+
+          # This solves it by manually defining the final location (try to solve the error above)
+          uri.port = 50075
+          uri      = uri.to_s + '&offset=0'
+
+          http   = EventMachine::HttpRequest.new(uri).aget
+          finish = proc { yield nil }
+          http.stream { |chunk| yield chunk }
+          http.callback &finish
+          http.errback &finish
+        end
+
+        # Saves the image file to the its final destination, based on the temporary file
+        # created by the server at data reception time.
+        #
+        # @param [String] id The image id.
+        # @param [File] tmp_file The temporary file descriptor.
+        # @param [String] format The image file format.
+        #
+        # @return [String, Integer] The generated file location URI and image file size.
+        #
+        # @raise [Duplicated] If the image file already exists.
+        #
+        def save(id, tmp_file, format)
+          @file = "#{id}.#{format}"
+          uri   = "hdfs://#{username}@#{host}:#{port}/#{base}/#{bucket}/#{file}"
+          size  = tmp_file.size
+
+          path     = generate_uri('op=CREATE&overwrite=true')
+          http     = EventMachine::HttpRequest.new(path).put
+          location = URI(http.response_header['LOCATION'])
+
+          location.hostname = host
+          raise Duplicated, "The image file #{uri} already exists" if file_exists?(false)
+          STDERR.puts "COPYING!!"
+
+          EventMachine::HttpRequest.new(location).put :file => tmp_file.path
+          [uri, size]
+        end
+
+        # Deletes the image file from its location.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def delete
+          uri = generate_uri('op=DELETE&recursive=true')
+          EventMachine::HttpRequest.new(uri).delete
+        end
+
+        # Check if the image file exists.
+        #
+        # @param [True, False] raise_exc (true) If it should raise exception or return
+        #   true/false whether the file exists or not.
+        #
+        # @return [True, False] If raise_exc is false, return true/false whether the
+        #   file exists or not.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def file_exists?(raise_exc=true)
+          uri   = generate_uri('op=GETFILESTATUS')
+          req   = Net::HTTP::Get.new(uri.request_uri)
+          res   = Net::HTTP.new(uri.hostname, uri.port).request(req)
+          exist = res.is_a? Net::HTTPSuccess
+          raise NotFound, "No image file found at #{uri}" if raise_exc && !exist
+          exist
+        end
+
+        def generate_uri(params)
+          URI("http://#{host}:#{port}/#{base}/#{bucket}/#{file}?#{params}&user.name=#{username}")
+        end
+      end
+
+    end
+  end
+end

data/lib/image/store/http.rb

@@ -0,0 +1,78 @@
+require "em-synchrony"
+require "em-synchrony/em-http"
+
+module Visor
+  module Image
+    module Store
+
+      # The HTTP backend store.
+      #
+      # This class handles the management of image files located in a remote HTTP location,
+      # based on a URI like *'http://www.domain.com/path-to-image-file'*.
+      #
+      # Useful for point an image to the last release of some distro, like:
+      #   'http://www.ubuntu.com/start-download?distro=server&bits=64&release=latest'
+      #
+      class HTTP
+        include Visor::Common::Exception
+
+        attr_accessor :uri, :config
+
+        # Initializes a new HTTP store client object.
+        #
+        # @param [String] uri The URI of the file location.
+        # @param [Hash] config (nil) A set of configurations for the wanted store,
+        #   loaded from VISoR configuration file.
+        #
+        # @return [Object] An instantiated HTTP store object ready to use.
+        #
+        def initialize(uri, config=nil)
+          @uri    = uri
+          @config = config
+        end
+
+        # Returns the image file to clients, streamed in chunks.
+        #
+        # @return [Object] Yields the file, a chunk at time.
+        #
+        def get
+          http   = EventMachine::HttpRequest.new(uri).get
+          finish = proc { yield nil }
+
+          http.stream { |chunk| yield chunk }
+          http.callback &finish
+          http.errback &finish
+        end
+
+        # Check if the image file exists. This will follow redirection to a
+        # nested deepness of 5 levels. It will also try to follow the location header
+        # if any.
+        #
+        # Also, after finding the real location of the HTTP file, it will parse the file
+        # metadata, most properly the size and checksum, based on URL headers.
+        #
+        # @return [String] The discovered file checksum and size.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def file_exists?(raise_exc=true)
+          http = EventMachine::HttpRequest.new(uri, connect_timeout: 2, redirects: 5).head
+
+          if location = http.response_header['LOCATION']
+            http = EventMachine::HttpRequest.new(location, connect_timeout: 2).head
+          end
+
+          exist    = (http.response_header.status == 200)
+          length   = http.response_header['CONTENT_LENGTH']
+          size     = length.nil? ? nil : length.to_i
+          etag     = http.response_header['ETAG']
+          checksum = etag.nil? ? '' : etag.gsub('"', '')
+
+          raise NotFound, "No image file found at #{uri}" if raise_exc && !exist
+          [exist, size, checksum]
+        end
+      end
+
+    end
+  end
+end

data/lib/image/store/lunacloud.rb

@@ -0,0 +1,126 @@
+require 'uri'
+require 's3restful'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+
+module Visor
+  module Image
+    module Store
+
+      # The Lunacloud backend store.
+      #
+      # This class handles the management of image files located in the Lunacloud storage system,
+      # based on a URI like *lunacloud://<access_key>:<secret_key>@<host>:<port>/<bucket>/<image>*.
+      #
+      class Lunacloud
+        include Visor::Common::Exception
+
+        attr_accessor :uri, :config, :access_key, :secret_key, :bucket, :file, :host, :port
+
+        # Initializes a new Lunacloud store client object. Lunacloud credentials are loaded from the URI,
+        # on GET and DELETE operations, or from the configuration file for POST and PUT operation.
+        #
+        # @param [String] uri The URI of the file location.
+        # @param config [Hash] A set of configurations for the wanted store, loaded from
+        #   VISoR configuration file.
+        #
+        # @return [Object] An instantiated Lunacloud store object ready to use.
+        #
+        def initialize(uri, config)
+          @uri    = URI(uri)
+          @config = config[:lunacloud]
+
+          if @uri.scheme
+            @access_key = @uri.user
+            @secret_key = @uri.password
+            @bucket     = @uri.path.split('/')[1]
+            @file       = @uri.path.split('/')[2]
+            @host       = @uri.host
+            @port       = @uri.port
+          else
+            @access_key = @config[:access_key]
+            @secret_key = @config[:secret_key]
+            @bucket     = @config[:bucket]
+            @host       = @config[:host]
+            @port       = @config[:port]
+          end
+        end
+
+        # Returns a Happening library S3 Lunacloud compatible connection object.
+        #
+        # @return [Happening::S3::Item] A new Lunacloud connection object.
+        #
+        def connection
+          S3restful::S3::Item.new(bucket, file, server: host, port: port, protocol: 'http',
+                                  aws_access_key_id: access_key, aws_secret_access_key: secret_key)
+        end
+
+        # Returns the image file to clients, streamed in chunks.
+        #
+        # @return [Object] Yields the file, a chunk at time.
+        #
+        def get
+          s3     = connection.aget
+          finish = proc { yield nil }
+
+          s3.stream { |chunk| yield chunk }
+          s3.callback &finish
+          s3.errback &finish
+        end
+
+        # Saves the image file to the its final destination, based on the temporary file
+        # created by the server at data reception time.
+        #
+        # @param [String] id The image id.
+        # @param [File] tmp_file The temporary file descriptor.
+        # @param [String] format The image file format.
+        #
+        # @return [String, Integer] The generated file location URI and image file size.
+        #
+        # @raise [Duplicated] If the image file already exists.
+        #
+        def save(id, tmp_file, format)
+          @file = "#{id}.#{format}"
+          uri   = "lunacloud://#{access_key}:#{secret_key}@#{host}:#{port}/#{bucket}/#{file}"
+          size  = tmp_file.size
+
+          raise Duplicated, "The image file #{fp} already exists" if file_exists?(false)
+          STDERR.puts "COPYING!!"
+
+          connection.store tmp_file.path
+
+          [uri, size]
+        end
+
+        # Deletes the image file from its location.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def delete
+          connection.delete
+        end
+
+        # Check if the image file exists.
+        #
+        # @param [True, False] raise_exc (true) If it should raise exception or return
+        #   true/false whether the file exists or not.
+        #
+        # @return [True, False] If raise_exc is false, return true/false whether the
+        #   file exists or not.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def file_exists?(raise_exc=true)
+          exist   = nil
+          error   = proc { exist = false }
+          success = proc { |res| exist = true if res.response_header.status == 200 }
+
+          connection.head(on_error: error, on_success: success)
+          raise NotFound, "No image file found at #{uri}" if raise_exc && !exist
+          exist
+        end
+      end
+
+    end
+  end
+end

data/lib/image/store/s3.rb

@@ -0,0 +1,121 @@
+require 'uri'
+require 's3restful'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+
+module Visor
+  module Image
+    module Store
+
+      # The Amazon Simple Storage (S3) backend store.
+      #
+      # This class handles the management of image files located in the S3 storage system,
+      # based on a URI like *s3://<access_key>:<secret_key@s3.amazonaws.com/<bucket>/<image>*.
+      #
+      class S3
+        include Visor::Common::Exception
+
+        attr_accessor :uri, :config, :access_key, :secret_key, :bucket, :file
+
+        # Initializes a new S3 store client object. S3 credentials are loaded from the URI,
+        # on GET and DELETE operations, or from the configuration file for POST and PUT operation.
+        #
+        # @param [String] uri The URI of the file location.
+        # @param config [Hash] A set of configurations for the wanted store, loaded from
+        #   VISoR configuration file.
+        #
+        # @return [Object] An instantiated S3 store object ready to use.
+        #
+        def initialize(uri, config)
+          @uri    = URI(uri)
+          @config = config[:s3]
+
+          if @uri.scheme
+            @access_key = @uri.user
+            @secret_key = @uri.password
+            @bucket     = @uri.path.split('/')[1]
+            @file       = @uri.path.split('/')[2]
+          else
+            @access_key = @config[:access_key]
+            @secret_key = @config[:secret_key]
+            @bucket     = @config[:bucket]
+          end
+        end
+
+        # Returns a Happening library S3 connection object.
+        #
+        # @return [Happening::S3::Item] A new S3 connection object.
+        #
+        def connection
+          S3restful::S3::Item.new(bucket, file, aws_access_key_id: access_key, aws_secret_access_key: secret_key)
+        end
+
+        # Returns the image file to clients, streamed in chunks.
+        #
+        # @return [Object] Yields the file, a chunk at time.
+        #
+        def get
+          s3     = connection.aget
+          finish = proc { yield nil }
+
+          s3.stream { |chunk| yield chunk }
+          s3.callback &finish
+          s3.errback &finish
+        end
+
+        # Saves the image file to the its final destination, based on the temporary file
+        # created by the server at data reception time.
+        #
+        # @param [String] id The image id.
+        # @param [File] tmp_file The temporary file descriptor.
+        # @param [String] format The image file format.
+        #
+        # @return [String, Integer] The generated file location URI and image file size.
+        #
+        # @raise [Duplicated] If the image file already exists.
+        #
+        def save(id, tmp_file, format)
+          @file = "#{id}.#{format}"
+          uri   = "s3://#{access_key}:#{secret_key}@s3.amazonaws.com/#{bucket}/#{file}"
+          size  = tmp_file.size
+
+          raise Duplicated, "The image file #{fp} already exists" if file_exists?(false)
+          STDERR.puts "COPYING!!"
+
+          connection.store tmp_file.path
+
+          [uri, size]
+        end
+
+        # Deletes the image file from its location.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def delete
+          connection.delete
+        end
+
+        # Check if the image file exists.
+        #
+        # @param [True, False] raise_exc (true) If it should raise exception or return
+        #   true/false whether the file exists or not.
+        #
+        # @return [True, False] If raise_exc is false, return true/false whether the
+        #   file exists or not.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def file_exists?(raise_exc=true)
+          exist   = nil
+          error   = proc { exist = false }
+          success = proc { |res| exist = true if res.response_header.status == 200 }
+
+          connection.head(on_error: error, on_success: success)
+          raise NotFound, "No image file found at #{uri}" if raise_exc && !exist
+          exist
+        end
+      end
+
+    end
+  end
+end