cloudkeeper 1.0.0

data/lib/cloudkeeper/errors/network_connection_error.rb

@@ -0,0 +1,5 @@
+module Cloudkeeper
+  module Errors
+    class NetworkConnectionError < StandardError; end
+  end
+end

data/lib/cloudkeeper/errors/nginx_error.rb

@@ -0,0 +1,5 @@
+module Cloudkeeper
+  module Errors
+    class NginxError < StandardError; end
+  end
+end

data/lib/cloudkeeper/errors/no_such_file_error.rb

@@ -0,0 +1,5 @@
+module Cloudkeeper
+  module Errors
+    class NoSuchFileError < StandardError; end
+  end
+end

data/lib/cloudkeeper/errors/not_implemented_error.rb

@@ -0,0 +1,5 @@
+module Cloudkeeper
+  module Errors
+    class NotImplementedError < StandardError; end
+  end
+end

data/lib/cloudkeeper/errors/parsing.rb

@@ -0,0 +1,10 @@
+module Cloudkeeper
+  module Errors
+    module Parsing
+      autoload :ParsingError, 'cloudkeeper/errors/parsing/parsing_error'
+      autoload :InvalidApplianceHashError, 'cloudkeeper/errors/parsing/invalid_appliance_hash_error'
+      autoload :InvalidImageHashError, 'cloudkeeper/errors/parsing/invalid_image_hash_error'
+      autoload :InvalidImageListHashError, 'cloudkeeper/errors/parsing/invalid_image_list_hash_error'
+    end
+  end
+end

data/lib/cloudkeeper/errors/parsing/invalid_appliance_hash_error.rb

@@ -0,0 +1,7 @@
+module Cloudkeeper
+  module Errors
+    module Parsing
+      class InvalidApplianceHashError < ParsingError; end
+    end
+  end
+end

data/lib/cloudkeeper/errors/parsing/invalid_image_hash_error.rb

@@ -0,0 +1,7 @@
+module Cloudkeeper
+  module Errors
+    module Parsing
+      class InvalidImageHashError < ParsingError; end
+    end
+  end
+end

data/lib/cloudkeeper/errors/parsing/invalid_image_list_hash_error.rb

@@ -0,0 +1,7 @@
+module Cloudkeeper
+  module Errors
+    module Parsing
+      class InvalidImageListHashError < ParsingError; end
+    end
+  end
+end

data/lib/cloudkeeper/errors/parsing/parsing_error.rb

@@ -0,0 +1,7 @@
+module Cloudkeeper
+  module Errors
+    module Parsing
+      class ParsingError < StandardError; end
+    end
+  end
+end

data/lib/cloudkeeper/errors/permission_denied_error.rb

@@ -0,0 +1,5 @@
+module Cloudkeeper
+  module Errors
+    class PermissionDeniedError < StandardError; end
+  end
+end

data/lib/cloudkeeper/errors/standard_error.rb

@@ -0,0 +1,5 @@
+module Cloudkeeper
+  module Errors
+    class StandardError < ::StandardError; end
+  end
+end

data/lib/cloudkeeper/managers.rb

@@ -0,0 +1,7 @@
+module Cloudkeeper
+  module Managers
+    autoload :ImageListManager, 'cloudkeeper/managers/image_list_manager'
+    autoload :ImageManager, 'cloudkeeper/managers/image_manager'
+    autoload :ApplianceManager, 'cloudkeeper/managers/appliance_manager'
+  end
+end

data/lib/cloudkeeper/managers/appliance_manager.rb

@@ -0,0 +1,152 @@
+module Cloudkeeper
+  module Managers
+    class ApplianceManager
+      attr_reader :backend_connector, :image_list_manager, :acceptable_formats
+
+      IMAGE_UPDATE_ATTRIBUTES = ['hv:uri', 'sl:checksum:sha512', 'hv:size'].freeze
+
+      def initialize
+        @backend_connector = Cloudkeeper::BackendConnector.new
+        @image_list_manager = Cloudkeeper::Managers::ImageListManager.new
+        @acceptable_formats = Cloudkeeper::Settings[:formats].map(&:to_sym)
+      end
+
+      def synchronize_appliances
+        logger.debug 'Running appliance synchronization...'
+        backend_connector.pre_action
+
+        backend_image_lists = backend_connector.image_lists
+        image_list_manager.download_image_lists
+
+        sync_expired_image_lists
+        sync_new_image_lists(backend_image_lists)
+        sync_old_image_lists(backend_image_lists)
+
+        backend_connector.post_action
+      rescue Cloudkeeper::Errors::BackendError => ex
+        abort ex.message
+      end
+
+      private
+
+      def sync_expired_image_lists
+        logger.debug 'Removing appliances from expired image lists...'
+        image_list_manager.image_lists.each_value do |image_list|
+          backend_connector.remove_image_list image_list if image_list.expired?
+        end
+      end
+
+      def sync_new_image_lists(backend_image_lists)
+        logger.debug 'Registering appliances from new image lists...'
+        add_list = image_list_manager.image_lists.keys - backend_image_lists
+        add_list.each do |image_list_identifier|
+          image_list_manager.image_lists[image_list_identifier].appliances.each_value { |appliance| add_appliance appliance }
+        end
+      end
+
+      def sync_old_image_lists(backend_image_lists)
+        logger.debug 'Synchronizing registered appliances...'
+        sync_list = image_list_manager.image_lists.keys & backend_image_lists
+        sync_list.each { |image_list_identifier| sync_image_list image_list_identifier }
+      end
+
+      def sync_image_list(image_list_identifier)
+        backend_appliances = backend_connector.appliances image_list_identifier
+        image_list_appliances = image_list_manager.image_lists[image_list_identifier].appliances
+
+        remove_appliances backend_appliances, image_list_appliances
+        add_appliances backend_appliances, image_list_appliances
+        update_appliances backend_appliances, image_list_appliances
+      end
+
+      def remove_appliances(backend_appliances, image_list_appliances)
+        logger.debug 'Removing previously registered appliances...'
+        remove_list = backend_appliances.keys - image_list_appliances.keys
+        remove_list.each { |appliance_identifier| backend_connector.remove_appliance image_list_appliances[appliance_identifier] }
+      end
+
+      def add_appliances(backend_appliances, image_list_appliances)
+        logger.debug 'Registering new appliances...'
+        add_list = image_list_appliances.keys - backend_appliances.keys
+        add_list.each { |appliance_identifier| add_appliance image_list_appliances[appliance_identifier] }
+      end
+
+      def update_appliances(backend_appliances, image_list_appliances)
+        logger.debug 'Updating appliances...'
+        update_list = backend_appliances.keys & image_list_appliances.keys
+        update_list.each do |appliance_identifier|
+          image_list_appliance = image_list_appliances[appliance_identifier]
+          backend_appliance = backend_appliances[appliance_identifier]
+
+          image_update = update_image?(image_list_appliance, backend_appliance)
+          image_list_appliance.image = nil unless image_update
+          update_appliance image_list_appliance if image_update || update_metadata?(image_list_appliance, backend_appliance)
+        end
+      end
+
+      def clean_image_files(appliance)
+        return unless appliance && appliance.image
+
+        logger.debug "Cleaning downloaded image files for appliance #{appliance.identifier.inspect}"
+        appliance.image.image_files.each { |image_file| clean_image_file image_file.file }
+      rescue ::IOError => ex
+        logger.warn "Appliance cleanup error: #{ex.message}"
+      end
+
+      def clean_image_file(filename)
+        File.delete(filename) if File.exist?(filename)
+      end
+
+      def update_image?(image_list_appliance, backend_appliance)
+        image_list_attributes = image_list_appliance.attributes
+        backend_attributes = backend_appliance.attributes
+
+        IMAGE_UPDATE_ATTRIBUTES.reduce(false) { |red, elem| red || (image_list_attributes[elem] != backend_attributes[elem]) }
+      end
+
+      def update_metadata?(image_list_appliance, backend_appliance)
+        image_list_appliance.attributes != backend_appliance.attributes
+      end
+
+      def update_appliance(appliance)
+        modify_appliance :update_appliance, appliance
+      end
+
+      def add_appliance(appliance)
+        modify_appliance :add_appliance, appliance
+      end
+
+      def modify_appliance(method, appliance)
+        prepare_image!(appliance) if appliance.image
+        backend_connector.send method, appliance
+      rescue Cloudkeeper::Errors::Image::DownloadError, Cloudkeeper::Errors::Image::ConversionError => ex
+        logger.error "Image preparation error: #{ex.message}"
+      rescue Cloudkeeper::Errors::Appliance::PropagationError => ex
+        logger.error "Appliance propagation error: #{ex.message}"
+      ensure
+        clean_image_files appliance
+      end
+
+      def prepare_image!(appliance)
+        image_file = Cloudkeeper::Managers::ImageManager.download_image(appliance.image.uri)
+        appliance.image.add_image_file image_file
+        return if acceptable_formats.include? image_file.format
+
+        convert_image! appliance, image_file
+      end
+
+      def convert_image!(appliance, image_file)
+        format = acceptable_formats.find { |acceptable_format| image_file.respond_to? "to_#{acceptable_format}".to_sym }
+        unless format
+          raise Cloudkeeper::Errors::Image::Format::NoRequiredFormatAvailableError,
+                "image #{image.inspect} cannot be converted to any acceptable format"
+        end
+
+        appliance.image.add_image_file image_file.send("to_#{format}".to_sym)
+      rescue Cloudkeeper::Errors::Image::Format::NoRequiredFormatAvailableError, Cloudkeeper::Errors::CommandExecutionError,
+             Cloudkeeper::Errors::ArgumentError, ::IOError => ex
+        raise Cloudkeeper::Errors::Image::ConversionError, ex
+      end
+    end
+  end
+end

data/lib/cloudkeeper/managers/image_list_manager.rb

@@ -0,0 +1,88 @@
+require 'tmpdir'
+require 'faraday'
+require 'zaru'
+require 'openssl'
+require 'json'
+
+module Cloudkeeper
+  module Managers
+    class ImageListManager
+      attr_reader :image_lists, :openssl_store
+
+      def initialize
+        @image_lists = {}
+
+        @openssl_store = OpenSSL::X509::Store.new
+        @openssl_store.add_path Cloudkeeper::Settings[:'ca-dir'] if Cloudkeeper::Settings[:'ca-dir']
+      end
+
+      def download_image_lists
+        logger.debug 'Downloading fresh image lists...'
+        Dir.mktmpdir('cloudkeeper') do |dir|
+          urls = Cloudkeeper::Settings[:'image-lists']
+          retrieve_image_lists urls, dir
+        end
+      end
+
+      private
+
+      def retrieve_image_lists(urls, dir)
+        urls.each do |url|
+          begin
+            image_list = convert_image_list(load_image_list(download_image_list(url, dir)))
+            image_lists[image_list.identifier] = image_list
+          rescue Cloudkeeper::Errors::ImageList::DownloadError, Cloudkeeper::Errors::ImageList::VerificationError,
+                 Cloudkeeper::Errors::Parsing::ParsingError => ex
+            logger.warn "Image list #{url} couldn't be donwloaded\n#{ex.message}"
+            next
+          end
+        end
+      end
+
+      def download_image_list(url, dir)
+        logger.debug "Downloading image list from #{url.inspect}"
+        Cloudkeeper::Utils::URL.check!(url)
+        uri = URI.parse url
+
+        filename = generate_filename(uri, dir)
+        response = make_request uri
+
+        if response.success?
+          File.write filename, response.body
+          return filename
+        end
+
+        raise Cloudkeeper::Errors::ImageList::RetrievalError,
+              "couldn't download image list from url #{url.inspect}\n#{response.to_hash.inspect}"
+      rescue Cloudkeeper::Errors::ImageList::RetrievalError, Cloudkeeper::Errors::InvalidURLError, ::IOError,
+             ::Faraday::ConnectionFailed => ex
+        raise Cloudkeeper::Errors::ImageList::DownloadError, ex
+      end
+
+      def make_request(uri)
+        conn = Faraday.new url: uri
+        conn.get
+      end
+
+      def generate_filename(uri, dir)
+        File.join(dir, Zaru.sanitize!("#{uri.host}#{uri.path}"))
+      end
+
+      def convert_image_list(image_list_hash)
+        Cloudkeeper::Entities::ImageList.from_hash image_list_hash
+      end
+
+      def load_image_list(file)
+        pkcs7 = OpenSSL::PKCS7.read_smime(File.read(file))
+        verify_image_list!(pkcs7, file)
+
+        JSON.parse pkcs7.data
+      end
+
+      def verify_image_list!(pkcs7, file)
+        raise Cloudkeeper::Errors::ImageList::VerificationError, "image list #{file.inspect} cannot be verified" \
+          unless pkcs7.verify([], openssl_store)
+      end
+    end
+  end
+end

data/lib/cloudkeeper/managers/image_manager.rb

@@ -0,0 +1,82 @@
+require 'net/http'
+
+module Cloudkeeper
+  module Managers
+    class ImageManager
+      extend Cloudkeeper::Entities::ImageFormats::Ova
+
+      FORMATS = {
+        qcow2: /qemu qcow image/i,
+        ova: /posix tar archive/i,
+        vmdk: /vmware4 disk image/i,
+        raw: /boot sector/i
+      }.freeze
+
+      class << self
+        def format(file)
+          check_file!(file)
+          recognize_format(file)
+        rescue Cloudkeeper::Errors::CommandExecutionError, Cloudkeeper::Errors::NoSuchFileError,
+               Cloudkeeper::Errors::PermissionDeniedError, Cloudkeeper::Errors::Image::Format::NoFormatRecognizedError,
+               Cloudkeeper::Errors::Image::Format::Ova::OvaFormatError => ex
+          raise Cloudkeeper::Errors::Image::Format::RecognitionError, ex, "Cannot recognize image format for file #{file.inspect}"
+        end
+
+        def file_description(file)
+          Cloudkeeper::CommandExecutioner.execute('file', '-b', file)
+        end
+
+        def check_file!(file)
+          raise Cloudkeeper::Errors::NoSuchFileError, "No such file #{file.inspect}" unless File.exist?(file)
+          raise Cloudkeeper::Errors::PermissionDeniedError, "Cannot read file #{file.inspect}" unless File.readable?(file)
+        end
+
+        def recognize_format(file)
+          file_format_string = file_description(file)
+          FORMATS.each do |format, regex|
+            next unless regex =~ file_format_string
+
+            format_test_method = "#{format}?".to_sym
+            additional_test_result = respond_to?(format_test_method) ? send(format_test_method, file) : true
+
+            return format if additional_test_result
+          end
+
+          raise Cloudkeeper::Errors::Image::Format::NoFormatRecognizedError, "No image format recognized for file #{file.inspect}"
+        end
+
+        def download_image(url)
+          logger.debug "Downloading image from #{url.inspect}"
+          Cloudkeeper::Utils::URL.check!(url)
+
+          uri = URI.parse url
+          filename = generate_filename(uri)
+          retrieve_image(uri, filename)
+
+          Cloudkeeper::Entities::ImageFile.new filename, format(filename), Cloudkeeper::Utils::Checksum.compute(filename), true
+        rescue Cloudkeeper::Errors::InvalidURLError, Cloudkeeper::Errors::Image::Format::RecognitionError,
+               Cloudkeeper::Errors::ArgumentError, Cloudkeeper::Errors::NetworkConnectionError, ::IOError => ex
+          raise Cloudkeeper::Errors::Image::DownloadError, ex
+        end
+
+        def retrieve_image(uri, filename)
+          Net::HTTP.start(uri.host, uri.port) do |http|
+            request = Net::HTTP::Get.new(uri)
+
+            http.request(request) do |response|
+              response.value
+              open(filename, 'w') { |file| response.read_body { |chunk| file.write(chunk) } }
+            end
+          end
+        rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError, Net::HTTPBadResponse,
+               Net::HTTPServerException, Net::HTTPHeaderSyntaxError, Net::ProtocolError => ex
+          raise Cloudkeeper::Errors::NetworkConnectionError, ex
+        end
+
+        def generate_filename(uri)
+          File.join(Cloudkeeper::Settings[:'image-dir'], Zaru.sanitize!(File.basename(uri.path)))
+        end
+      end
+    end
+  end
+end

data/lib/cloudkeeper/nginx.rb

@@ -0,0 +1,5 @@
+module Cloudkeeper
+  module Nginx
+    autoload :HttpServer, 'cloudkeeper/nginx/http_server'
+  end
+end

data/lib/cloudkeeper/nginx/http_server.rb

@@ -0,0 +1,113 @@
+require 'webrick'
+require 'tempfile'
+require 'securerandom'
+require 'erb'
+require 'tilt/erb'
+
+module Cloudkeeper
+  module Nginx
+    class HttpServer
+      attr_reader :auth_file, :conf_file, :access_data
+
+      def initialize
+        @access_data = {}
+      end
+
+      def start(image_file)
+        logger.debug 'Starting NGINX server'
+        @access_data = {}
+        credentials = prepare_credentials
+        configuration = prepare_configuration File.dirname(image_file), File.basename(image_file)
+        prepare_configuration_file configuration
+        fill_access_data credentials, configuration
+
+        Cloudkeeper::CommandExecutioner.execute Cloudkeeper::Settings[:'nginx-binary'], '-c', conf_file.path
+      rescue Cloudkeeper::Errors::CommandExecutionError, ::IOError => ex
+        stop
+        raise Cloudkeeper::Errors::NginxError, ex
+      end
+
+      def stop
+        logger.debug 'Stopping NGINX server'
+        if conf_file
+          Cloudkeeper::CommandExecutioner.execute Cloudkeeper::Settings[:'nginx-binary'], '-s', 'stop', '-c', conf_file.path
+          conf_file.unlink
+        end
+
+        auth_file.unlink if auth_file
+        @access_data = {}
+      rescue Cloudkeeper::Errors::CommandExecutionError, ::IOError => ex
+        raise Cloudkeeper::Errors::NginxError, ex
+      end
+
+      private
+
+      def fill_access_data(credentials, configuration)
+        access_data.merge! credentials
+        access_data[:url] = "http://#{configuration[:ip_address]}:#{configuration[:port]}/#{configuration[:image_file]}"
+      end
+
+      def prepare_credentials
+        username = random_string
+        password = random_string
+
+        write_auth_file username, password
+
+        logger.debug("Prepared NGINX authentication file #{auth_file.path.inspect}: "\
+                     "username: #{username.inspect}, password: #{password.inspect}")
+
+        { username: username, password: password }
+      end
+
+      def write_auth_file(username, password)
+        @auth_file = Tempfile.new('cloudkeeper-nginx-auth')
+        passwd = WEBrick::HTTPAuth::Htpasswd.new(auth_file.path)
+        passwd.set_passwd(nil, username, password)
+        passwd.flush
+        auth_file.close
+      end
+
+      def prepare_configuration_file(configuration)
+        conf_content = prepare_configuration_file_content configuration
+        write_configuration_file conf_content
+
+        logger.debug("Prepared NGINX configuration file #{conf_file.path.inspect}:\n#{conf_content}")
+      end
+
+      def write_configuration_file(content)
+        @conf_file = Tempfile.new('cloudkeeper-nginx-conf')
+
+        conf_file.write content
+        conf_file.close
+      end
+
+      def prepare_configuration_file_content(configuration)
+        conf_template = Tilt::ERBTemplate.new(File.join(File.expand_path(File.dirname(__FILE__)), 'templates', 'nginx.conf.erb'))
+        conf_template.render(Object.new, configuration)
+      end
+
+      def prepare_configuration(root_dir, image_file)
+        nginx_configuration = {}
+        nginx_configuration[:error_log_file] = Cloudkeeper::Settings[:'nginx-error-log-file']
+        nginx_configuration[:access_log_file] = Cloudkeeper::Settings[:'nginx-access-log-file']
+        nginx_configuration[:pid_file] = Cloudkeeper::Settings[:'nginx-pid-file']
+        nginx_configuration[:auth_file] = auth_file.path
+        nginx_configuration[:root_dir] = root_dir
+        nginx_configuration[:image_file] = image_file
+        nginx_configuration[:ip_address] = Cloudkeeper::Settings[:'nginx-ip-address']
+        nginx_configuration[:port] = choose_port
+
+        logger.debug("NGINX configuration: #{nginx_configuration.inspect}")
+        nginx_configuration
+      end
+
+      def choose_port
+        rand(Cloudkeeper::Settings[:'nginx-min-port']..Cloudkeeper::Settings[:'nginx-max-port'])
+      end
+
+      def random_string
+        SecureRandom.uuid
+      end
+    end
+  end
+end