visor-image 0.0.1

data/lib/image/server.rb

@@ -0,0 +1,307 @@
+require 'goliath'
+require 'tempfile'
+require 'json'
+
+module Visor
+  module Image
+
+    # The VISoR Image Server. This supports all image metadata manipulation
+    # operations, dispatched to the VISoR Meta Server and the image files storage operations.
+    #
+    # *The Server API is a RESTful web service for image meta as follows:*
+    #
+    #   HEAD    /images/<id>    - Returns metadata about the image with the given id
+    #   GET     /images         - Returns a set of brief metadata about all public images
+    #   GET     /images/detail  - Returns a set of detailed metadata about all public images
+    #   GET     /images/<id>    - Returns image data and metadata for the image with the given id
+    #   POST    /images         - Stores a new image data and metadata and returns the registered metadata
+    #   PUT     /images/<id>    - Update image metadata and/or data for the image with the given id
+    #   DELETE  /images/<id>    - Delete the metadata and data of the image with the given id
+    #
+    # The Image is multi-format, most properly it supports response encoding in JSON and XML formats.
+    # It will auto negotiate and encode the response metadata and error messages in the proper format,
+    # based on the request *Accept* header, being it 'application/json' or 'application/xml'.
+    # If no Accept header is provided, Image will encode and render it as JSON by default.
+    #
+    # This server routes all metadata operations to the {Visor::Meta::Server Visor Meta Server}.
+    #
+    # The server will interact with the multiple supported backend store to manage the image files.
+    #
+    # *Currently we support the following store backend and operations:*
+    #
+    #   Amazon Simple Storage (s3)  - GET, POST, PUT, DELETE
+    #   Nimbus Cumulus (cumulus)    - GET, POST, PUT, DELETE
+    #   Eucalyptus Walrus (walrus)  - GET, POST, PUT, DELETE
+    #   Local Filesystem (file)     - GET, POST, PUT, DELETE
+    #   Remote HTTP (http)          - GET
+    #
+    class Server < Goliath::API
+
+      # Middleware
+      #
+      # Listen at /status for a heartbeat server message status
+      use Goliath::Rack::Heartbeat
+      # Auto parse and merge body and query parameters
+      use Goliath::Rack::Params
+      # Cleanup accepted media types
+      use Goliath::Rack::DefaultMimeType
+
+
+      # Routes
+      #
+
+      # @method head_image
+      # @overload head '/image/:id'
+      #
+      # Head metadata about the image with the given id, see {Visor::Image::HeadImage}.
+      #
+      # The image metadata is promptly passed as a set of HTTP headers, as the following example:
+      #
+      #   x-image-meta-_id:           <id>
+      #   x-image-meta-uri:           <uri>
+      #   x-image-meta-name:          <name>
+      #   x-image-meta-architecture:  <architecture>
+      #   x-image-meta-access:        <access>
+      #   x-image-meta-status:        <status>
+      #   x-image-meta-created_at:    <timestamp>
+      #
+      # @note Undefined attributes are ignored and not included into response's header. Also
+      #   any raised error will be passed through HTTP headers as expected.
+      #
+      # @param [String] id The wanted image _id.
+      #
+      # @example Retrieve the image metadata with id '19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d':
+      #   'HEAD /images/19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d'
+      #
+      # @return [HTTP Headers] The image data file.
+      #
+      # @raise [HTTP Error 404] If image not found.
+      # @raise [HTTP Error 500] On internal server error.
+      #
+      head '/images/:id', HeadImage
+
+
+      # @method get_all_brief
+      # @overload get '/images'
+      #
+      # Get brief information about all public images, see {Visor::Image::GetImages}.
+      #
+      #   { "images":
+      #     [
+      #       {
+      #         "_id":<_id>,
+      #         "uri":<uri>,
+      #         "name":<name>,
+      #         "architecture":<architecture>,
+      #         "type":<type>,
+      #         "format":<format>,
+      #         "store":<type>,
+      #         "size":<size>,
+      #         "created_at":<timestamp>
+      #       },
+      #       ...
+      #     ]
+      #   }
+      #
+      # The following options can be passed as query parameters, plus any other additional
+      # image attribute not defined in the schema.
+      #
+      # @param [String, Integer, Date] parameter The image attribute to filter results based on its value.
+      # @param [String] sort ("_id") The image attribute to sort results.
+      # @param [String] dir ("asc") The sorting order ("asc"/"desc").
+      #
+      # @example Retrieve all public images brief metadata:
+      #   'GET /images'
+      #
+      # @example Retrieve all public `x86_64` images brief metadata:
+      #   'GET /images?architecture=x86_64'
+      #
+      # @example Retrieve all public `.iso` images brief metadata, descending sorted by `size`:
+      #   'GET /images?format=iso&sort=size&dir=desc'
+      #
+      # @return [JSON, XML] The public images brief metadata.
+      #
+      # @raise [HTTP Error 404] If there is no public images.
+      # @raise [HTTP Error 500] On internal server error.
+      #
+      get '/images', GetImages
+
+
+      # @method get_all_detail
+      # @overload get '/images/detail'
+      #
+      # Get detailed information about all public images, see {Visor::Image::GetImagesDetail}.
+      #
+      #   { "images":
+      #     [
+      #       {
+      #         "_id":<_id>,
+      #         "uri":<uri>,
+      #         "name":<name>,
+      #         "architecture":<architecture>,
+      #         "access":<access>,
+      #         "status":<status>,
+      #         "size":<size>,
+      #         "type":<type>,
+      #         "format":<format>,
+      #         "store":<store>,
+      #         "created_at":<timestamp>
+      #         "updated_at":<timestamp>,
+      #         "kernel":<associated kernel>,
+      #         "ramdisk":<associated ramdisk>,
+      #       },
+      #       ...
+      #     ]
+      #   }
+      #
+      # The following options can be passed as query parameters, plus any other additional
+      # image attribute not defined in the schema.
+      #
+      # @param [String, Integer, Date] parameter The image attribute to filter results based on its value.
+      # @param [String] sort ("_id") The image attribute to sort results.
+      # @param [String] dir ("asc") The sorting order ("asc"/"desc").
+      #
+      # @example Retrieve all public images detailed metadata:
+      #   'GET /images/detail'
+      #
+      # @note Querying and ordering results are made as with #get_all_detail
+      #
+      # @return [JSON, XML] The public images detailed metadata.
+      #
+      # @raise [HTTP Error 404] If there is no public images.
+      # @raise [HTTP Error 500] On internal server error.
+      #
+      get '/images/detail', GetImagesDetail
+
+
+      # @method get_image
+      # @overload get '/images/:id'
+      #
+      # Get image data and detailed metadata for the given id, see {Visor::Image::GetImage}.
+      #
+      # The image data file is streamed as response's body. The server will return a 200 status code,
+      # with a special HTTP header, indicating that the response body will be streamed in chunks
+      # and connection shouldn't be closed until the transfer is complete.
+      #
+      # Also, the image metadata is promptly passed as a set of HTTP headers, as the following example:
+      #
+      #   x-image-meta-_id:           <id>
+      #   x-image-meta-uri:           <uri>
+      #   x-image-meta-name:          <name>
+      #   x-image-meta-architecture:  <architecture>
+      #   x-image-meta-access:        <access>
+      #   x-image-meta-status:        <status>
+      #   x-image-meta-created_at:    <timestamp>
+      #
+      # @note Undefined attributes are ignored and not included into response's header.
+      #
+      # @param [String] id The wanted image _id.
+      #
+      # @example Retrieve the image with id '19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d':
+      #   'GET /images/19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d'
+      #
+      # @return [HTTP Headers] The image data file.
+      # @return [HTTP Body] The image data file.
+      #
+      # @raise [HTTP Error 404] If image not found.
+      # @raise [HTTP Error 500] On internal server error.
+      #
+      get '/images/:id', GetImage
+
+
+      # @method post
+      # @overload post '/images'
+      #
+      # Post image data and metadata and returns the registered metadata, see {Visor::Image::PostImage}.
+      #
+      # The image metadata should be encoded as HTTP headers and passed with the request, as the following example:
+      #
+      #   x-image-meta-name:          Ubuntu 10.10 Desktop
+      #   x-image-meta-architecture:  i386
+      #   x-image-meta-store:         s3
+      #
+      # If wanted, the image data file should be streamed through the request body.
+      # The server will receive that data in chunks, buffering them to a properly secured temporary
+      # file, avoiding buffering all the data into memory. Server will then handle the upload of that
+      # data to the definitive store location, cleaning then the generated temporary file.
+      #
+      # Alternatively, a *x-image-meta-location* header can be passed, if the file is already stored in some
+      # provided location. If this header is present, no body content can be passed in the request.
+      #
+      # @return [JSON, XML] The already created image detailed metadata.
+      #
+      # @raise [HTTP Error 400] If the image metadata validation fails.
+      # @raise [HTTP Error 400] If the location header is present no file content can be provided.
+      # @raise [HTTP Error 400] If trying to post an image file to a HTTP backend.
+      # @raise [HTTP Error 400] If provided store is an unsupported store backend.
+      # @raise [HTTP Error 404] If no image data is found at the provided location.
+      # @raise [HTTP Error 409] If the provided image file already exists in the backend store.
+      # @raise [HTTP Error 500] On internal server error.
+      #
+      post '/images', PostImage
+
+
+      # @method put
+      # @overload put '/images/:id'
+      #
+      # Put image metadata and/or data for the image with the given id, see {Visor::Image::PutImage}.
+      #
+      # The image metadata should be encoded as HTTP headers and passed with the request, as the following example:
+      #
+      #   x-image-meta-name:          Ubuntu 10.10 Server
+      #   x-image-meta-architecture:  x86_64
+      #   x-image-meta-location:      http://www.ubuntu.com/start-download?distro=server&bits=64&release=latest
+      #
+      # If wanted, the image data file should be streamed through the request body.
+      # The server will receive that data in chunks, buffering them to a properly secured temporary
+      # file, avoiding buffering all the data into memory. Server will then handle the upload of that
+      # data to the definitive store location, cleaning then the generated temporary file.
+      #
+      # Alternatively, a *x-image-meta-location* header can be passed, if the file is already stored in some
+      # provided location. If this header is present, no body content can be passed in the request.
+      #
+      # @note Only images with status set to 'locked' or 'error' can be updated with an image data file.
+      #
+      # @return [JSON, XML] The already updated image detailed metadata.
+      #
+      # @raise [HTTP Error 400] If the image metadata validation fails.
+      # @raise [HTTP Error 400] If no headers neither body found for update.
+      # @raise [HTTP Error 400] If the location header is present no file content can be provided.
+      # @raise [HTTP Error 400] If trying to post an image file to a HTTP backend.
+      # @raise [HTTP Error 400] If provided store is an unsupported store backend.
+      # @raise [HTTP Error 404] If no image data is found at the provided location.
+      # @raise [HTTP Error 409] If trying to assign image file to a locked or uploading image.
+      # @raise [HTTP Error 409] If the provided image file already exists in the backend store.
+      # @raise [HTTP Error 500] On internal server error.
+      #
+      put '/images/:id', PutImage
+
+
+      # @method delete
+      # @overload delete '/images/:id'
+      #
+      # Delete the metadata and data of the image with the given id, see {Visor::Image::DeleteImage}.
+      #
+      # @param [String] id The image _id to delete.
+      #
+      # @example Delete the image with id '19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d':
+      #   'DELETE /images/19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d'
+      #
+      # @return [JSON, XML] The already deleted image detailed metadata.
+      #
+      # @raise [HTTP Error 404] If image meta or data not found.
+      # @raise [HTTP Error 403] If user does not have permission to manipulate the image file.
+      # @raise [HTTP Error 500] On internal server error.
+      #
+      delete '/images/:id', DeleteImage
+
+      delete '/all', DeleteAllImages
+
+      # Not Found
+      not_found('/') do
+        run Proc.new { |env| [404, {}, {code: 404, message: "Invalid operation or path."}.to_json] }
+      end
+    end
+
+  end
+end

data/lib/image/store/cumulus.rb

@@ -0,0 +1,126 @@
+require 'uri'
+require 's3restful'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+
+module Visor
+  module Image
+    module Store
+
+      # The Nimbus Cumulus (Cumulus) backend store.
+      #
+      # This class handles the management of image files located in the Cumulus storage system,
+      # based on a URI like *cumulus://<access_key>:<secret_key>@<host>:<port>/<bucket>/<image>*.
+      #
+      class Cumulus
+        include Visor::Common::Exception
+
+        attr_accessor :uri, :config, :access_key, :secret_key, :bucket, :file, :host, :port
+
+        # Initializes a new Cumulus store client object. Cumulus credentials are loaded from the URI,
+        # on GET and DELETE operations, or from the configuration file for POST and PUT operation.
+        #
+        # @param [String] uri The URI of the file location.
+        # @param config [Hash] A set of configurations for the wanted store, loaded from
+        #   VISoR configuration file.
+        #
+        # @return [Object] An instantiated Cumulus store object ready to use.
+        #
+        def initialize(uri, config)
+          @uri    = URI(uri)
+          @config = config[:cumulus]
+
+          if @uri.scheme
+            @access_key = @uri.user
+            @secret_key = @uri.password
+            @bucket     = @uri.path.split('/')[1]
+            @file       = @uri.path.split('/')[2]
+            @host       = @uri.host
+            @port       = @uri.port
+          else
+            @access_key = @config[:access_key]
+            @secret_key = @config[:secret_key]
+            @bucket     = @config[:bucket]
+            @host       = @config[:host]
+            @port       = @config[:port]
+          end
+        end
+
+        # Returns a Happening library S3 Cumulus compatible connection object.
+        #
+        # @return [Happening::S3::Item] A new Cumulus connection object.
+        #
+        def connection
+          S3restful::S3::Item.new(bucket, file, server: host, port: port, protocol: 'http',
+                                  aws_access_key_id: access_key, aws_secret_access_key: secret_key)
+        end
+
+        # Returns the image file to clients, streamed in chunks.
+        #
+        # @return [Object] Yields the file, a chunk at time.
+        #
+        def get
+          s3     = connection.aget
+          finish = proc { yield nil }
+
+          s3.stream { |chunk| yield chunk }
+          s3.callback &finish
+          s3.errback &finish
+        end
+
+        # Saves the image file to the its final destination, based on the temporary file
+        # created by the server at data reception time.
+        #
+        # @param [String] id The image id.
+        # @param [File] tmp_file The temporary file descriptor.
+        # @param [String] format The image file format.
+        #
+        # @return [String, Integer] The generated file location URI and image file size.
+        #
+        # @raise [Duplicated] If the image file already exists.
+        #
+        def save(id, tmp_file, format)
+          @file = "#{id}.#{format}"
+          uri   = "cumulus://#{access_key}:#{secret_key}@#{host}:#{port}/#{bucket}/#{file}"
+          size  = tmp_file.size
+
+          raise Duplicated, "The image file #{fp} already exists" if file_exists?(false)
+          STDERR.puts "COPYING!!"
+
+          connection.store tmp_file.path
+
+          [uri, size]
+        end
+
+        # Deletes the image file from its location.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def delete
+          connection.delete
+        end
+
+        # Check if the image file exists.
+        #
+        # @param [True, False] raise_exc (true) If it should raise exception or return
+        #   true/false whether the file exists or not.
+        #
+        # @return [True, False] If raise_exc is false, return true/false whether the
+        #   file exists or not.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def file_exists?(raise_exc=true)
+          exist   = nil
+          error   = proc { exist = false }
+          success = proc { |res| exist = true if res.response_header.status == 200 }
+
+          connection.head(on_error: error, on_success: success)
+          raise NotFound, "No image file found at #{uri}" if raise_exc && !exist
+          exist
+        end
+      end
+
+    end
+  end
+end

data/lib/image/store/file_system.rb

@@ -0,0 +1,119 @@
+require 'uri'
+
+module Visor
+  module Image
+    module Store
+
+      # The FileSystem backend store.
+      #
+      # This class handles the management of image files located in the local FileSystem,
+      # based on a URI like *file:///path/to/my_image.format*.
+      #
+      class FileSystem
+        include Visor::Common::Exception
+
+        # Size of the chunk to stream the files out.
+        CHUNKSIZE = 65536
+
+        attr_accessor :uri, :fp, :config
+
+        # Initializes a new FileSystem store client object.
+        #
+        # @param [String] uri The URI of the file location.
+        # @param config [Hash] A set of configurations for the wanted store, loaded from
+        #   VISoR configuration file.
+        #
+        # @return [Object] An instantiated FileSystem store object ready to use.
+        #
+        def initialize(uri, config)
+          @uri    = URI(uri)
+          @fp     = @uri.path
+          @config = config[:file]
+        end
+
+        # Returns the image file to clients, streamed in chunks.
+        #
+        # @return [Object] Yields the file, a chunk at time.
+        #
+        def get
+          file_exists?
+          open(fp, "rb") do |file|
+            yield file.read(CHUNKSIZE) until file.eof?
+            yield nil
+          end
+        end
+
+        # Saves the image file to the its final destination, based on the temporary file
+        # created by the server at data reception time.
+        #
+        # @param [String] id The image id.
+        # @param [File] tmp_file The temporary file descriptor.
+        # @param [String] format The image file format.
+        #
+        # @return [String, Integer] The generated file location URI and image file size.
+        #
+        # @raise [Duplicated] If the image file already exists.
+        #
+        def save(id, tmp_file, format)
+          dir  = File.expand_path config[:directory]
+          file = "#{id}.#{format}"
+          fp   = File.join(dir, file)
+          uri  = "file://#{fp}"
+          size = tmp_file.size
+
+          FileUtils.mkpath(dir) unless Dir.exists?(dir)
+          raise Duplicated, "The image file #{fp} already exists" if File.exists?(fp)
+          STDERR.puts "Copying image tempfile #{tmp_file.path} to definitive #{fp}"
+
+          tmp = File.open(tmp_file.path, "rb")
+          new = File.open(fp, "wb")
+
+          each_chunk(tmp, CHUNKSIZE) do |chunk|
+            new << chunk
+          end
+
+          [uri, size]
+        end
+
+        # Deletes the image file to from its location.
+        #
+        # @raise [Forbidden] If user does not have permission to manipulate the image file.
+        # @raise [NotFound] If the image file was not found.
+        #
+        def delete
+          file_exists?
+          begin
+            File.delete(fp)
+          rescue => e
+            raise Forbidden, "Error while trying to delete image file #{fp}: #{e.message}"
+          end
+        end
+
+        # Check if the image file exists.
+        #
+        # @raise [NotFound] If the image file was not found.
+        #
+        def file_exists?
+          raise NotFound, "No image file found at #{fp}" unless File.exists?(fp)
+        end
+
+        private
+
+        # Iterates over the image file yielding a chunk per reactor tick.
+        #
+        # @return [Object] Yielded image file chunk.
+        #
+        def each_chunk(file, chunk_size=CHUNKSIZE)
+          handler = lambda do
+            unless file.eof?
+              yield file.read(chunk_size)
+              EM.next_tick &handler
+            end
+          end
+          EM.next_tick &handler
+        end
+
+      end
+    end
+  end
+end