cloudkeeper 1.0.0

data/lib/cloudkeeper/cli.rb

@@ -0,0 +1,174 @@
+require 'thor'
+require 'yell'
+
+module Cloudkeeper
+  class CLI < Thor
+    class_option :'logging-level',
+                 required: true,
+                 default: Cloudkeeper::Settings['logging']['level'],
+                 type: :string,
+                 enum: Yell::Severities
+    class_option :'logging-file',
+                 default: Cloudkeeper::Settings['logging']['file'],
+                 type: :string,
+                 desc: 'File to write logs to'
+    class_option :debug,
+                 default: Cloudkeeper::Settings['debug'],
+                 type: :boolean,
+                 desc: 'Runs cloudkeeper in debug mode'
+
+    method_option :'image-lists',
+                  required: true,
+                  default: Cloudkeeper::Settings['image-lists'],
+                  type: :array,
+                  desc: 'List of image lists to sync against'
+    method_option :'ca-dir',
+                  required: false,
+                  default: Cloudkeeper::Settings['ca-dir'],
+                  type: :string,
+                  desc: 'CA directory'
+    method_option :authentication,
+                  default: Cloudkeeper::Settings['authentication'],
+                  type: :boolean,
+                  desc: 'Client <-> server authentication'
+    method_option :certificate,
+                  required: false,
+                  default: Cloudkeeper::Settings['certificate'],
+                  type: :string,
+                  desc: "Core's host certificate"
+    method_option :key,
+                  required: false,
+                  default: Cloudkeeper::Settings['key'],
+                  type: :string,
+                  desc: "Core's host key"
+    method_option :'image-dir',
+                  required: true,
+                  default: Cloudkeeper::Settings['image-dir'],
+                  type: :string,
+                  desc: 'Directory to store images to'
+    method_option :'qemu-img-binary',
+                  required: true,
+                  default: Cloudkeeper::Settings['binaries']['qemu-img'],
+                  type: :string,
+                  desc: 'Path to qemu-img binary (image conversion)'
+    method_option :'nginx-binary',
+                  default: Cloudkeeper::Settings['binaries']['nginx'],
+                  type: :string,
+                  desc: 'Path to nginx binary (HTTP server)'
+    method_option :'remote-mode',
+                  default: Cloudkeeper::Settings['remote-mode'],
+                  type: :boolean,
+                  desc: 'Remote mode starts HTTP server (NGINX) and serves images to backend via HTTP'
+    method_option :'nginx-error-log-file',
+                  default: Cloudkeeper::Settings['nginx']['error-log-file'],
+                  type: :string,
+                  desc: 'NGINX error log file'
+    method_option :'nginx-access-log-file',
+                  default: Cloudkeeper::Settings['nginx']['access-log-file'],
+                  type: :string,
+                  desc: 'NGINX access log file'
+    method_option :'nginx-pid-file',
+                  default: Cloudkeeper::Settings['nginx']['pid-file'],
+                  type: :string,
+                  desc: 'NGINX pid file'
+    method_option :'nginx-ip-address',
+                  default: Cloudkeeper::Settings['nginx']['ip-address'],
+                  type: :string,
+                  desc: 'IP address NGINX can listen on'
+    method_option :'nginx-min-port',
+                  default: Cloudkeeper::Settings['nginx']['min-port'],
+                  type: :numeric,
+                  desc: 'Minimal port NGINX can listen on'
+    method_option :'nginx-max-port',
+                  default: Cloudkeeper::Settings['nginx']['max-port'],
+                  type: :numeric,
+                  desc: 'Maximal port NGINX can listen on'
+    method_option :'backend-endpoint',
+                  required: true,
+                  default: Cloudkeeper::Settings['backend']['endpoint'],
+                  type: :string,
+                  desc: "Backend's gRPC endpoint"
+    method_option :'backend-certificate',
+                  required: false,
+                  default: Cloudkeeper::Settings['backend']['certificate'],
+                  type: :string,
+                  desc: "Backend's certificate"
+    method_option :formats,
+                  required: true,
+                  default: Cloudkeeper::Settings['formats'],
+                  type: :array,
+                  desc: 'List of acceptable formats images can be converted to'
+
+    desc 'sync', 'Runs synchronization process'
+    def sync
+      initialize_sync options
+      Cloudkeeper::Managers::ApplianceManager.new.synchronize_appliances
+    rescue Cloudkeeper::Errors::InvalidConfigurationError => ex
+      abort ex.message
+    end
+
+    desc 'version', 'Prints cloudkeeper version'
+    def version
+      $stdout.puts Cloudkeeper::VERSION
+    end
+
+    default_task :sync
+
+    private
+
+    def initialize_sync(options)
+      initialize_configuration options
+      validate_configuration!
+      initialize_logger
+      logger.debug "Cloudkeeper 'sync' called with parameters: #{Cloudkeeper::Settings.to_hash.inspect}"
+    end
+
+    def initialize_configuration(options)
+      Cloudkeeper::Settings.clear
+      Cloudkeeper::Settings.merge! options.to_hash
+    end
+
+    def validate_configuration!
+      validate_configuration_group! :authentication,
+                                    [:certificate, :key, :'backend-certificate'],
+                                    'Authentication configuration missing'
+      validate_configuration_group! :'remote-mode',
+                                    [:'nginx-binary', :'nginx-error-log-file', :'nginx-access-log-file', :'nginx-pid-file',
+                                     :'nginx-ip-address', :'nginx-min-port', :'nginx-max-port'],
+                                    'NGINX configuration missing'
+    end
+
+    def validate_configuration_group!(flag, required_options, error_message)
+      return unless Cloudkeeper::Settings[flag]
+
+      raise Cloudkeeper::Errors::InvalidConfigurationError, error_message unless all_options_available(required_options)
+    end
+
+    def all_options_available(required_options)
+      required_options.reduce(true) { |acc, elem| Cloudkeeper::Settings[elem] && acc }
+    end
+
+    def initialize_logger
+      Cloudkeeper::Settings[:'logging-level'] = 'DEBUG' if Cloudkeeper::Settings[:debug]
+
+      logging_file = Cloudkeeper::Settings[:'logging-file']
+      logging_level = Cloudkeeper::Settings[:'logging-level']
+
+      Yell.new :stdout, name: Object, level: logging_level.downcase, format: Yell::DefaultFormat
+      Object.send :include, Yell::Loggable
+
+      setup_file_logger(logging_file) if logging_file
+
+      logger.debug 'Running in debug mode...'
+    end
+
+    def setup_file_logger(logging_file)
+      unless (File.exist?(logging_file) && File.writable?(logging_file)) || File.writable?(File.dirname(logging_file))
+        logger.error "File #{logging_file} isn't writable"
+        return
+      end
+
+      logger.adapter :file, logging_file
+    end
+  end
+end

data/lib/cloudkeeper/command_executioner.rb

@@ -0,0 +1,22 @@
+module Cloudkeeper
+  class CommandExecutioner
+    class << self
+      def execute(*args)
+        command = Mixlib::ShellOut.new(*args)
+        logger.debug "Executing command: #{command.command.inspect}"
+        command.run_command
+
+        if command.error?
+          raise Cloudkeeper::Errors::CommandExecutionError, "Command #{command.command.inspect} terminated with an error: " \
+                                                            "#{command.stderr}"
+        end
+
+        command.stdout
+      end
+
+      def list_archive(archive)
+        execute('tar', '-t', '-f', archive).lines.map(&:chomp)
+      end
+    end
+  end
+end

data/lib/cloudkeeper/entities.rb

@@ -0,0 +1,11 @@
+module Cloudkeeper
+  module Entities
+    autoload :ImageFile, 'cloudkeeper/entities/image_file'
+    autoload :Image, 'cloudkeeper/entities/image'
+    autoload :Appliance, 'cloudkeeper/entities/appliance'
+    autoload :ImageList, 'cloudkeeper/entities/image_list'
+    autoload :ImageFormats, 'cloudkeeper/entities/image_formats'
+    autoload :Convertables, 'cloudkeeper/entities/convertables'
+    autoload :Conversions, 'cloudkeeper/entities/conversions'
+  end
+end

data/lib/cloudkeeper/entities/appliance.rb

@@ -0,0 +1,84 @@
+module Cloudkeeper
+  module Entities
+    class Appliance
+      attr_accessor :identifier, :description, :mpuri, :title, :group, :ram, :core, :version, :architecture
+      attr_accessor :operating_system, :image, :attributes, :vo, :expiration_date, :image_list_identifier
+
+      REJECTED_ATTRIBUTES = [:vo, :expiration, :image_list_identifier].freeze
+
+      def initialize(identifier, mpuri, vo, expiration_date, image_list_identifier, title = '', description = '', group = '',
+                     ram = 1024, core = 1, version = '', architecture = '', operating_system = '', image = nil, attributes = {})
+        if identifier.blank? || \
+           mpuri.blank? || \
+           vo.blank? || \
+           expiration_date.blank? || \
+           image_list_identifier.blank?
+          raise Cloudkeeper::Errors::ArgumentError, 'identifier, mpuri, vo, expiration_date and image_list_identifier ' \
+                                                    'cannot be nil nor empty'
+        end
+
+        @identifier = identifier
+        @description = description
+        @mpuri = mpuri
+        @title = title
+        @group = group
+        @ram = ram
+        @core = core
+        @version = version
+        @architecture = architecture
+        @operating_system = operating_system
+        @image = image
+        @attributes = attributes
+        @vo = vo
+        @expiration_date = expiration_date
+        @image_list_identifier = image_list_identifier
+      end
+
+      class << self
+        def from_hash(appliance_hash)
+          appliance_hash.deep_symbolize_keys!
+          appliance = populate_appliance appliance_hash
+          appliance.image = Image.from_hash(appliance_hash)
+
+          appliance
+        end
+
+        def populate_appliance(appliance_hash)
+          raise Cloudkeeper::Errors::Parsing::InvalidApplianceHashError, 'invalid appliance hash' if appliance_hash.blank?
+
+          appliance = Appliance.new appliance_hash[:'dc:identifier'],
+                                    appliance_hash[:'ad:mpuri'],
+                                    appliance_hash[:vo],
+                                    appliance_hash[:expiration],
+                                    appliance_hash[:image_list_identifier],
+                                    appliance_hash[:'dc:title'],
+                                    appliance_hash[:'dc:description'],
+                                    appliance_hash[:'ad:group'],
+                                    appliance_hash[:'ad:ram_recommended'],
+                                    appliance_hash[:'ad:core_recommended'],
+                                    appliance_hash[:'hv:version'],
+                                    appliance_hash[:'sl:arch']
+
+          construct_os_name!(appliance, appliance_hash)
+          populate_attributes!(appliance, appliance_hash)
+
+          appliance
+        rescue Cloudkeeper::Errors::ArgumentError => ex
+          raise Cloudkeeper::Errors::Parsing::InvalidApplianceHashError, ex, "appliance hash #{appliance_hash.inspect} " \
+                                                                             "doesn't contain all the necessary data"
+        end
+
+        def construct_os_name!(appliance, appliance_hash)
+          appliance.operating_system = appliance_hash[:'sl:os'].to_s
+          appliance.operating_system = "#{appliance.operating_system} #{appliance_hash[:'sl:osname']}".strip
+          appliance.operating_system = "#{appliance.operating_system} #{appliance_hash[:'sl:osversion']}".strip
+        end
+
+        def populate_attributes!(appliance, appliance_hash)
+          appliance_hash.reject! { |k, _v| REJECTED_ATTRIBUTES.include? k }
+          appliance.attributes = appliance_hash.map { |k, v| [k.to_s, v.to_s] }.to_h
+        end
+      end
+    end
+  end
+end

data/lib/cloudkeeper/entities/conversions.rb

@@ -0,0 +1,48 @@
+module Cloudkeeper
+  module Entities
+    module Conversions
+      private
+
+      def convert_image(image)
+        image_file = acceptable_image_file image
+
+        CloudkeeperGrpc::Image.new mode: :LOCAL, location: image_file.file, format: image_file.format.upcase,
+                                   checksum: image_file.checksum, size: image.size.to_i, uri: image.uri
+      end
+
+      def convert_appliance(appliance, image_proto)
+        CloudkeeperGrpc::Appliance.new identifier: appliance.identifier.to_s, description: appliance.description.to_s,
+                                       mpuri: appliance.mpuri.to_s, title: appliance.title.to_s, group: appliance.group.to_s,
+                                       ram: appliance.ram.to_i, core: appliance.core.to_i, version: appliance.version.to_s,
+                                       architecture: appliance.architecture.to_s, operating_system: appliance.operating_system.to_s,
+                                       vo: appliance.vo.to_s, image: image_proto, expiration_date: appliance.expiration_date.to_i,
+                                       image_list_identifier: appliance.image_list_identifier.to_s, attributes: appliance.attributes
+      end
+
+      def convert_image_proto(image_proto)
+        return nil unless image_proto
+
+        Cloudkeeper::Entities::Image.new image_proto.uri, image_proto.checksum, image_proto.size
+      end
+
+      def convert_appliance_proto(appliance_proto, image)
+        Cloudkeeper::Entities::Appliance.new appliance_proto.identifier, appliance_proto.mpuri, appliance_proto.vo,
+                                             Time.at(appliance_proto.expiration_date).to_datetime,
+                                             appliance_proto.image_list_identifier, appliance_proto.title,
+                                             appliance_proto.description, appliance_proto.group, appliance_proto.ram,
+                                             appliance_proto.core, appliance_proto.version, appliance_proto.architecture,
+                                             appliance_proto.operating_system, image, appliance_proto.attributes.to_h
+      end
+
+      def acceptable_image_file(image)
+        image_format = (image.available_formats & Cloudkeeper::Settings[:formats].map(&:to_sym).sort).first
+        unless image_format
+          raise Cloudkeeper::Errors::Image::Format::NoRequiredFormatAvailableError, 'image is not available in any of the ' \
+                                                                                    'required formats'
+        end
+
+        image.image_file image_format
+      end
+    end
+  end
+end

data/lib/cloudkeeper/entities/convertables.rb

@@ -0,0 +1,8 @@
+module Cloudkeeper
+  module Entities
+    module Convertables
+      autoload :Convertable, 'cloudkeeper/entities/convertables/convertable'
+      autoload :Ova, 'cloudkeeper/entities/convertables/ova'
+    end
+  end
+end

data/lib/cloudkeeper/entities/convertables/convertable.rb

@@ -0,0 +1,63 @@
+require 'digest'
+
+module Cloudkeeper
+  module Entities
+    module Convertables
+      module Convertable
+        CONVERT_OUTPUT_FORMATS = [:raw, :qcow2, :vmdk].freeze
+
+        def self.convert_output_formats
+          CONVERT_OUTPUT_FORMATS
+        end
+
+        FORMAT_REGEX = /^to_(?<format>#{convert_output_formats.join('|')})$/
+
+        def self.included(base)
+          raise Cloudkeeper::Errors::Convertables::ConvertabilityError, "#{base.inspect} cannot become a convertable" \
+            unless base.method_defined?(:file) && base.method_defined?(:format)
+
+          super
+        end
+
+        def method_missing(method, *arguments, &block)
+          result = method.to_s.match(FORMAT_REGEX)
+          return convert(result[:format]) if result && result[:format]
+
+          super
+        end
+
+        def respond_to_missing?(method, *)
+          method =~ FORMAT_REGEX || super
+        end
+
+        private
+
+        def convert(output_format)
+          logger.debug "Converting file #{file.inspect} from #{format.inspect} to #{output_format.inspect}"
+          return self if output_format.to_sym == format.to_sym
+
+          converted_file = File.join(File.dirname(file), "#{File.basename(file, '.*')}.#{output_format}")
+          run_convert_command(output_format, converted_file)
+
+          image_file converted_file, output_format
+        end
+
+        def run_convert_command(output_format, converted_file)
+          Cloudkeeper::CommandExecutioner.execute(Cloudkeeper::Settings[:'qemu-img-binary'],
+                                                  'convert',
+                                                  '-f',
+                                                  format.to_s,
+                                                  '-O',
+                                                  output_format.to_s,
+                                                  file,
+                                                  converted_file)
+        end
+
+        def image_file(converted_file, output_format)
+          Cloudkeeper::Entities::ImageFile.new converted_file, output_format.to_sym,
+                                               Cloudkeeper::Utils::Checksum.compute(converted_file), false
+        end
+      end
+    end
+  end
+end

data/lib/cloudkeeper/entities/convertables/ova.rb

@@ -0,0 +1,54 @@
+module Cloudkeeper
+  module Entities
+    module Convertables
+      module Ova
+        CONVERT_OUTPUT_FORMATS = [:raw, :qcow2].freeze
+
+        def self.convert_output_formats
+          CONVERT_OUTPUT_FORMATS
+        end
+
+        def self.extended(base)
+          raise Cloudkeeper::Errors::Convertables::ConvertabilityError, "#{base.inspect} cannot become OVA convertable" \
+            unless base.class.included_modules.include?(Cloudkeeper::Entities::Convertables::Convertable)
+
+          super
+        end
+
+        def to_vmdk
+          image_file(extract_disk, :vmdk)
+        end
+
+        def to_ova
+          self
+        end
+
+        private
+
+        def convert(output_format)
+          logger.debug "Converting file #{file.inspect} from #{format.inspect} to #{output_format.inspect}"
+          vmdk_image = to_vmdk
+          final_image = vmdk_image.send("to_#{output_format}".to_sym)
+          File.delete vmdk_image.file
+
+          final_image
+        end
+
+        def extract_disk
+          archived_disk = disk_file
+          disk_directory = File.dirname(file)
+          Cloudkeeper::CommandExecutioner.execute('tar', '-x', '-f', file, '-C', disk_directory, archived_disk)
+          File.join(disk_directory, archived_disk)
+        end
+
+        def archive_files
+          Cloudkeeper::CommandExecutioner.list_archive file
+        end
+
+        def disk_file
+          archive_files.each { |file| return file if Cloudkeeper::Entities::ImageFormats::Ova::VMDK_REGEX =~ file }
+        end
+      end
+    end
+  end
+end