cloudkeeper 1.5.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e57ce8c830c735521f025d8f1c2840e6855439f
-  data.tar.gz: 3b49c31527aaed3e26c1272908fd1ef653919889
+  metadata.gz: 4c6230319495e64370c48ade72d90d2b43ce0e5e
+  data.tar.gz: dfc5256e110fe7cd8098f64271ace0a193e7a607
 SHA512:
-  metadata.gz: 0e311d61ee7aef14d2a519099ecb16f5c035b471bd7b2dd4f23385bbff21f617ab162c880855c57a7eadfed48a09b6744a0afd23ab0b51b18992a8eb3767e23d
-  data.tar.gz: 400b857a285479f84cea18c0012b727846594f2758375d40dd5c1d9ca9b2fad096603ff13ef519fdca567de9827038d0ee6e661b90ae24fe9509584eee745940
+  metadata.gz: 430fd633d0d538fcfc488ed6d83b7ef3121de74fd0a5a3a90d43dca7309a34fe04e05a841a3cf1f4651a7e8d7262618444445823f50c96f9d2b875d373a59417
+  data.tar.gz: f38f6430d1c203d08c31298a2409acfe3ccb360dc967e2cbf40fffb6585f09b2d67c3e53c08ba5242b5f966df99e7dda1907b39cdb97acb89f0788cb9c9f9515

data/.circleci/config.yml

@@ -6,25 +6,28 @@ jobs:
         - /.*/
     docker:
       - image: docker:stable
-    working_directory: /root/cloudkeeper
+    working_directory: /root/working_directory
     steps:
       - run: apk add --no-cache git openssh
       - checkout
       - setup_remote_docker
+      - run: |
+          git config --global --replace-all versionsort.prereleasesuffix ".alpha"
+          git config --global --add versionsort.prereleasesuffix ".beta"
       - run: |
           TAG=${CIRCLE_TAG#v}
           BRANCH=${TAG/%.*/.x}
           VERSION=${TAG}
-          LATEST=$(git tag --sort=-refname | head -n 1)
+          LATEST=$(git tag --sort=-version:refname | head -n 1)
 
           docker login -u $DOCKER_USER -p $DOCKER_PASS
 
-          docker build --build-arg branch=$BRANCH --build-arg version="$VERSION" -t cloudkeeper/cloudkeeper:$TAG .
-          docker push cloudkeeper/cloudkeeper:$TAG
+          docker build --build-arg branch=$BRANCH --build-arg version="$VERSION" -t $DOCKERHUB_ORGANIZATION/$CIRCLE_PROJECT_REPONAME:$TAG ./$DOCKERFILE_DIR
+          docker push $DOCKERHUB_ORGANIZATION/$CIRCLE_PROJECT_REPONAME:$TAG
 
           if [ "$LATEST" == "$CIRCLE_TAG" ]; then
-            docker tag cloudkeeper/cloudkeeper:$TAG cloudkeeper/cloudkeeper:latest
-            docker push cloudkeeper/cloudkeeper:latest
+            docker tag $DOCKERHUB_ORGANIZATION/$CIRCLE_PROJECT_REPONAME:$TAG $DOCKERHUB_ORGANIZATION/$CIRCLE_PROJECT_REPONAME:latest
+            docker push $DOCKERHUB_ORGANIZATION/$CIRCLE_PROJECT_REPONAME:latest
           fi
 deployment:
   fake_deploy_for_cci2:

data/.rubocop.yml

@@ -1,7 +1,7 @@
 require: rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.0
+  TargetRubyVersion: 2.2
   Exclude:
     - 'vendor/**/*'
     - 'lib/cloudkeeper_grpc/*'
@@ -12,6 +12,14 @@ Metrics/LineLength:
 Style/Documentation:
   Enabled: false
 
+Style/FormatStringToken:
+  Exclude:
+    - 'lib/cloudkeeper/utils/date.rb'
+
+Style/MixinUsage:
+  Exclude:
+    - 'spec/cloudkeeper/entities/convertables/ova_spec.rb'
+
 Metrics/MethodLength:
   Max: 15
   Exclude:

data/README.md

@@ -4,8 +4,9 @@ cloudkeeper is an AppDB <-> cloud synchronization utility
 [![Travis](https://img.shields.io/travis/the-cloudkeeper-project/cloudkeeper.svg?style=flat-square)](http://travis-ci.org/the-cloudkeeper-project/cloudkeeper)
 [![Gemnasium](https://img.shields.io/gemnasium/the-cloudkeeper-project/cloudkeeper.svg?style=flat-square)](https://gemnasium.com/the-cloudkeeper-project/cloudkeeper)
 [![Gem](https://img.shields.io/gem/v/cloudkeeper.svg?style=flat-square)](https://rubygems.org/gems/cloudkeeper)
-[![Code Climate](https://img.shields.io/codeclimate/github/the-cloudkeeper-project/cloudkeeper.svg?style=flat-square)](https://codeclimate.com/github/the-cloudkeeper-project/cloudkeeper)
+[![Code Climate](https://img.shields.io/codeclimate/maintainability/the-cloudkeeper-project/cloudkeeper.svg?style=flat-square)](https://codeclimate.com/github/the-cloudkeeper-project/cloudkeeper)
 [![DockerHub](https://img.shields.io/badge/docker-ready-blue.svg?style=flat-square)](https://hub.docker.com/r/cloudkeeper/cloudkeeper/)
+[![DOI](https://img.shields.io/badge/dynamic/json.svg?label=DOI&colorB=0D7EBE&prefix=&suffix=&query=$.doi&uri=https%3A%2F%2Fzenodo.org%2Fapi%2Frecords%2F891886&style=flat-square)](https://zenodo.org/record/891886)
 
 ## What does cloudkeeper do?
 cloudkeeper is able to read image lists provided by EGI AppDB, parse their content and decide what cloud appliances should be added, updated or removed from managed cloud. During the addition and update cloudkeeper is able to download an appliance's image and convert it to the format supported by the managed cloud.
@@ -74,7 +75,8 @@ Usage:
   cloudkeeper sync --backend-endpoint=BACKEND-ENDPOINT --external-tools-execution-timeout=N --formats=one two three --image-dir=IMAGE-DIR --image-lists=one two three --qemu-img-binary=QEMU-IMG-BINARY
 
 Options:
-  --image-lists=one two three                        # List of image lists to sync against
+  [--image-lists=one two three]                      # List of image lists to sync against
+  [--image-lists-file=IMAGE-LISTS-FILE]              # File containing list of image lists to sync against
   [--ca-dir=CA-DIR]                                  # CA directory
                                                      # Default: /etc/grid-security/certificates/
   [--authentication], [--no-authentication]          # Client <-> server authentication

data/cloudkeeper.gemspec

@@ -30,27 +30,27 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'diffy', '~> 3.1'
+  spec.add_development_dependency 'grpc-tools', '>= 1.1', '<= 1.2.5'
+  spec.add_development_dependency 'pry', '~> 0.10'
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'rspec-collection_matchers', '~> 1.1'
   spec.add_development_dependency 'rubocop', '~> 0.48'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.15'
-  spec.add_development_dependency 'rspec-collection_matchers', '~> 1.1'
   spec.add_development_dependency 'simplecov', '~> 0.12'
-  spec.add_development_dependency 'pry', '~> 0.10'
   spec.add_development_dependency 'vcr', '~> 3.0'
   spec.add_development_dependency 'webmock', '~> 3.0'
-  spec.add_development_dependency 'diffy', '~> 3.1'
-  spec.add_development_dependency 'grpc-tools', '>= 1.1', '<= 1.2.5'
 
-  spec.add_runtime_dependency 'thor', '~> 0.19'
-  spec.add_runtime_dependency 'yell', '~> 2.0'
+  spec.add_runtime_dependency 'activesupport', '>= 4.0', '< 6.0'
+  spec.add_runtime_dependency 'faraday', '~> 0.11'
+  spec.add_runtime_dependency 'grpc', '~> 1.8'
   spec.add_runtime_dependency 'mixlib-shellout', '~> 2.2'
-  spec.add_runtime_dependency 'grpc', '>= 1.1', '<= 1.2.5'
   spec.add_runtime_dependency 'settingslogic', '~> 2.0'
-  spec.add_runtime_dependency 'zaru', '~> 0.1'
-  spec.add_runtime_dependency 'activesupport', '>= 4.0', '< 6.0'
+  spec.add_runtime_dependency 'thor', '~> 0.19'
   spec.add_runtime_dependency 'tilt', '~> 2.0'
-  spec.add_runtime_dependency 'faraday', '~> 0.11'
+  spec.add_runtime_dependency 'yell', '~> 2.0'
+  spec.add_runtime_dependency 'zaru', '~> 0.1'
 
   spec.required_ruby_version = '>= 2.2.0'
 end

data/config/cloudkeeper.yml

@@ -1,5 +1,6 @@
 cloudkeeper:
   image-lists: # List of image lists to sync against
+  image-lists-file: # File containing list of image lists to sync against
   ca-dir: /etc/grid-security/certificates/ # CA directory
   authentication: false # core (client) <-> backend (server) authentication (certificate, key and backend-certificate options)
   certificate: /etc/grid-security/hostcert.pem # Core's host certificate

data/lib/cloudkeeper/backend_connector.rb

@@ -56,7 +56,7 @@ module Cloudkeeper
         grpc_client.appliances(
           CloudkeeperGrpc::ImageListIdentifier.new(image_list_identifier: image_list_identifier),
           return_op: true
-        )
+        ), raise_exception: true
       ) do |response|
         response.inject({}) do |acc, elem|
           image = convert_image_proto(elem.image)

data/lib/cloudkeeper/cli.rb

@@ -23,10 +23,13 @@ module Cloudkeeper
                  desc: 'Runs cloudkeeper in debug mode'
 
     method_option :'image-lists',
-                  required: true,
                   default: Cloudkeeper::Settings['image-lists'],
                   type: :array,
                   desc: 'List of image lists to sync against'
+    method_option :'image-lists-file',
+                  default: Cloudkeeper::Settings['image-lists-file'],
+                  type: :string,
+                  desc: 'File containing list of image lists to sync against'
     method_option :'ca-dir',
                   required: false,
                   default: Cloudkeeper::Settings['ca-dir'],
@@ -168,6 +171,7 @@ module Cloudkeeper
       validate_configuration_group! %i[remote-mode nginx-proxy-ip-address],
                                     %i[nginx-proxy-port],
                                     'NGINX proxy configuration missing'
+      validate_contradictory_options! %i[image-lists image-lists-file], one_required: true
     end
 
     def validate_configuration_group!(flags, required_options, error_message)
@@ -176,6 +180,16 @@ module Cloudkeeper
       raise Cloudkeeper::Errors::InvalidConfigurationError, error_message unless all_options_available(required_options)
     end
 
+    def validate_contradictory_options!(flags, options)
+      filled = flags.select { |flag| Cloudkeeper::Settings[flag] }
+      if filled.count > 1
+        raise Cloudkeeper::Errors::InvalidConfigurationError, "Following options cannot be used together: #{filled.join(', ')}"
+      end
+
+      return unless options[:one_required]
+      raise Cloudkeeper::Errors::InvalidConfigurationError, "One of the options #{flags.join(', ')} required" if filled.empty?
+    end
+
     def all_options_available(required_options)
       required_options.reduce(true) { |acc, elem| Cloudkeeper::Settings[elem] && acc }
     end

data/lib/cloudkeeper/entities/appliance.rb

@@ -4,7 +4,7 @@ module Cloudkeeper
       attr_accessor :identifier, :description, :mpuri, :title, :group, :ram, :core, :version, :architecture
       attr_accessor :operating_system, :image, :attributes, :vo, :expiration_date, :image_list_identifier
 
-      REJECTED_ATTRIBUTES = %i[vo expiration image_list_identifier].freeze
+      REJECTED_ATTRIBUTES = %i[vo image_list_identifier].freeze
 
       def initialize(identifier, mpuri, vo, expiration_date, image_list_identifier, title = '', description = '', group = '',
                      ram = 1024, core = 1, version = '', architecture = '', operating_system = '', image = nil, attributes = {})
@@ -34,6 +34,10 @@ module Cloudkeeper
         @image_list_identifier = image_list_identifier
       end
 
+      def expired?
+        expiration_date < Time.now
+      end
+
       class << self
         def from_hash(appliance_hash)
           appliance_hash.deep_symbolize_keys!
@@ -46,19 +50,7 @@ module Cloudkeeper
         def populate_appliance(appliance_hash)
           raise Cloudkeeper::Errors::Parsing::InvalidApplianceHashError, 'invalid appliance hash' if appliance_hash.blank?
 
-          appliance = Appliance.new appliance_hash[:'dc:identifier'],
-                                    appliance_hash[:'ad:mpuri'],
-                                    appliance_hash[:vo],
-                                    appliance_hash[:expiration],
-                                    appliance_hash[:image_list_identifier],
-                                    appliance_hash[:'dc:title'],
-                                    appliance_hash[:'dc:description'],
-                                    appliance_hash[:'ad:group'],
-                                    appliance_hash[:'hv:ram_minimum'],
-                                    appliance_hash[:'hv:core_minimum'],
-                                    appliance_hash[:'hv:version'],
-                                    appliance_hash[:'sl:arch']
-
+          appliance = construct_appliance(appliance_hash)
           construct_os_name!(appliance, appliance_hash)
           populate_attributes!(appliance, appliance_hash)
 
@@ -68,6 +60,21 @@ module Cloudkeeper
                                                                              "doesn't contain all the necessary data"
         end
 
+        def construct_appliance(appliance_hash)
+          Appliance.new appliance_hash[:'dc:identifier'],
+                        appliance_hash[:'ad:mpuri'],
+                        appliance_hash[:vo],
+                        Cloudkeeper::Utils::Date.parse(appliance_hash[:'dc:date:expires']),
+                        appliance_hash[:image_list_identifier],
+                        appliance_hash[:'dc:title'],
+                        appliance_hash[:'dc:description'],
+                        appliance_hash[:'ad:group'],
+                        appliance_hash[:'hv:ram_minimum'],
+                        appliance_hash[:'hv:core_minimum'],
+                        appliance_hash[:'hv:version'],
+                        appliance_hash[:'sl:arch']
+        end
+
         def construct_os_name!(appliance, appliance_hash)
           appliance.operating_system = appliance_hash[:'sl:os'].to_s
           appliance.operating_system = "#{appliance.operating_system} #{appliance_hash[:'sl:osname']}".strip

data/lib/cloudkeeper/entities/image_formats/ova.rb

@@ -17,19 +17,20 @@ module Cloudkeeper
         end
 
         def ova_structure?(files)
-          check_count! files
-          has_ovf = has_vmdk = false
+          check_file_count! files
 
-          files.each do |file|
-            has_ovf ||= OVF_REGEX =~ file
-            has_vmdk ||= VMDK_REGEX =~ file
-            break if has_ovf && has_vmdk
-          end
+          vmdk_count = files.select { |file| VMDK_REGEX =~ file }.count
+          ovf_count = files.select { |file| OVF_REGEX =~ file }.count
 
-          has_ovf && has_vmdk
+          raise Cloudkeeper::Errors::Image::Format::Ova::InvalidArchiveError, 'Archive contains multiple drives (VMDK files)' \
+            if vmdk_count > 1
+          raise Cloudkeeper::Errors::Image::Format::Ova::InvalidArchiveError, 'Archive contains multiple descriptors (OVF files)' \
+            if ovf_count > 1
+
+          vmdk_count == 1 && ovf_count == 1
         end
 
-        def check_count!(files)
+        def check_file_count!(files)
           return unless files.count > ARCHIVE_MAX_FILES
 
           raise Cloudkeeper::Errors::Image::Format::Ova::InvalidArchiveError, "Too many files in archive: #{files.count}. "\

data/lib/cloudkeeper/entities/image_list.rb

@@ -5,8 +5,6 @@ module Cloudkeeper
     class ImageList
       attr_accessor :identifier, :creation_date, :expiration_date, :description, :title, :source, :appliances
 
-      DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'.freeze
-
       def initialize(identifier, expiration_date, creation_date = nil, source = '', title = '', description = '', appliances = {})
         raise Cloudkeeper::Errors::ArgumentError, 'identifier cannot be nil nor empty' if identifier.blank? || expiration_date.blank?
 
@@ -26,7 +24,7 @@ module Cloudkeeper
       end
 
       def expired?
-        expiration_date < DateTime.now
+        expiration_date < Time.now
       end
 
       class << self
@@ -40,11 +38,12 @@ module Cloudkeeper
           image_list
         end
 
-        def prepare_appliance_hash(image_hash, endorser, expiration, vo, image_list_identifier)
+        def prepare_appliance_hash(image_hash, endorser, vo, image_list_identifier)
           appliance_hash = {}
 
           appliance_hash = image_hash[:'hv:image'] if image_hash && image_hash.key?(:'hv:image')
-          appliance_hash.merge!(vo: vo, expiration: expiration, image_list_identifier: image_list_identifier)
+          appliance_hash[:vo] = vo
+          appliance_hash[:image_list_identifier] = image_list_identifier
           appliance_hash.merge!(endorser[:'hv:x509']) if endorser && endorser.key?(:'hv:x509')
 
           appliance_hash
@@ -54,8 +53,8 @@ module Cloudkeeper
           raise Cloudkeeper::Errors::Parsing::InvalidImageListHashError, 'invalid image list hash' if image_list_hash.blank?
 
           ImageList.new image_list_hash[:'dc:identifier'],
-                        parse_date(image_list_hash[:'dc:date:expires']),
-                        parse_date(image_list_hash[:'dc:date:created']),
+                        Cloudkeeper::Utils::Date.parse(image_list_hash[:'dc:date:expires']),
+                        Cloudkeeper::Utils::Date.parse(image_list_hash[:'dc:date:created']),
                         image_list_hash[:'dc:source'],
                         image_list_hash[:'dc:title'],
                         image_list_hash[:'dc:description']
@@ -64,17 +63,12 @@ module Cloudkeeper
                                                                              "doesn't contain all the necessary data"
         end
 
-        def parse_date(date)
-          date.blank? ? '' : DateTime.strptime(date, DATE_FORMAT)
-        end
-
         def populate_appliances!(image_list, image_list_hash)
-          expiration = parse_date(image_list_hash[:'dc:date:expires'])
           vo = image_list_hash[:'ad:vo']
           endorser = image_list_hash[:'hv:endorser']
 
           image_list_hash[:'hv:images'].each do |image_hash|
-            appliance = Appliance.from_hash(prepare_appliance_hash(image_hash, endorser, expiration, vo, image_list.identifier))
+            appliance = Appliance.from_hash(prepare_appliance_hash(image_hash, endorser, vo, image_list.identifier))
             image_list.add_appliance appliance
           end
         end

data/lib/cloudkeeper/managers/appliance_manager.rb

@@ -1,9 +1,9 @@
 module Cloudkeeper
   module Managers
     class ApplianceManager
-      attr_reader :backend_connector, :image_list_manager, :acceptable_formats
+      include Cloudkeeper::Utils::Appliance
 
-      IMAGE_UPDATE_ATTRIBUTES = ['hv:version', 'sl:checksum:sha512', 'hv:size'].freeze
+      attr_reader :backend_connector, :image_list_manager, :acceptable_formats
 
       def initialize
         @backend_connector = Cloudkeeper::BackendConnector.new
@@ -41,7 +41,14 @@ module Cloudkeeper
         add_list = image_list_manager.image_lists.keys - backend_image_lists
         logger.debug "Image lists to register: #{add_list.inspect}"
         add_list.each do |image_list_identifier|
-          image_list_manager.image_lists[image_list_identifier].appliances.each_value { |appliance| add_appliance appliance }
+          image_list_manager.image_lists[image_list_identifier].appliances.each_value do |appliance|
+            if appliance.expired?
+              log_expired appliance, 'Skipping expired appliance'
+              next
+            end
+
+            add_appliance appliance
+          end
         end
       end
 
@@ -73,7 +80,15 @@ module Cloudkeeper
         logger.debug 'Registering new appliances...'
         add_list = image_list_appliances.keys - backend_appliances.keys
         logger.debug "Appliances to register: #{add_list.inspect}"
-        add_list.each { |appliance_identifier| add_appliance image_list_appliances[appliance_identifier] }
+        add_list.each do |appliance_identifier|
+          appliance = image_list_appliances[appliance_identifier]
+          if appliance.expired?
+            log_expired appliance, 'Skipping expired appliance'
+            next
+          end
+
+          add_appliance appliance
+        end
       end
 
       def update_appliances(backend_appliances, image_list_appliances)
@@ -84,40 +99,18 @@ module Cloudkeeper
           image_list_appliance = image_list_appliances[appliance_identifier]
           backend_appliance = backend_appliances[appliance_identifier]
 
+          if image_list_appliance.expired?
+            log_expired image_list_appliance, 'Removing expired appliance'
+            backend_connector.remove_appliance image_list_appliance
+            next
+          end
+
           image_update = update_image?(image_list_appliance, backend_appliance)
           image_list_appliance.image = nil unless image_update
           update_appliance image_list_appliance if image_update || update_metadata?(image_list_appliance, backend_appliance)
         end
       end
 
-      def clean_image_files(appliance)
-        return unless appliance && appliance.image
-
-        logger.debug "Cleaning downloaded image files for appliance #{appliance.identifier.inspect}"
-        appliance.image.image_files.each { |image_file| clean_image_file image_file.file }
-      rescue ::IOError => ex
-        logger.warn "Appliance cleanup error: #{ex.message}"
-      end
-
-      def clean_image_file(filename)
-        File.delete(filename) if File.exist?(filename)
-      end
-
-      def update_image?(image_list_appliance, backend_appliance)
-        image_list_attributes = image_list_appliance.attributes
-        backend_attributes = backend_appliance.attributes
-
-        IMAGE_UPDATE_ATTRIBUTES.reduce(false) { |red, elem| red || (image_list_attributes[elem] != backend_attributes[elem]) }
-      end
-
-      def update_metadata?(image_list_appliance, backend_appliance)
-        image_list_appliance.attributes != backend_appliance.attributes
-      end
-
-      def update_appliance(appliance)
-        modify_appliance :update_appliance, appliance
-      end
-
       def add_appliance(appliance)
         modify_appliance :add_appliance, appliance
       end
@@ -132,27 +125,6 @@ module Cloudkeeper
       ensure
         clean_image_files appliance
       end
-
-      def prepare_image!(appliance)
-        image_file = Cloudkeeper::Managers::ImageManager.download_image(appliance.image.uri)
-        appliance.image.add_image_file image_file
-        return if acceptable_formats.include? image_file.format
-
-        convert_image! appliance, image_file
-      end
-
-      def convert_image!(appliance, image_file)
-        format = acceptable_formats.find { |acceptable_format| image_file.respond_to? "to_#{acceptable_format}".to_sym }
-        unless format
-          raise Cloudkeeper::Errors::Image::Format::NoRequiredFormatAvailableError,
-                "image #{image.inspect} cannot be converted to any acceptable format"
-        end
-
-        appliance.image.add_image_file image_file.send("to_#{format}".to_sym)
-      rescue Cloudkeeper::Errors::Image::Format::NoRequiredFormatAvailableError, Cloudkeeper::Errors::CommandExecutionError,
-             Cloudkeeper::Errors::ArgumentError, ::IOError => ex
-        raise Cloudkeeper::Errors::Image::ConversionError, ex
-      end
     end
   end
 end

data/lib/cloudkeeper/managers/image_list_manager.rb

@@ -19,7 +19,7 @@ module Cloudkeeper
       def download_image_lists
         logger.debug 'Downloading fresh image lists...'
         Dir.mktmpdir('cloudkeeper') do |dir|
-          urls = Cloudkeeper::Settings[:'image-lists']
+          urls = Cloudkeeper::Settings[:'image-lists'] || File.read(Cloudkeeper::Settings[:'image-lists-file']).split("\n")
           retrieve_image_lists urls, dir
         end
       end

data/lib/cloudkeeper/utils.rb

@@ -3,5 +3,7 @@ module Cloudkeeper
     autoload :Hash, 'cloudkeeper/utils/hash'
     autoload :Checksum, 'cloudkeeper/utils/checksum'
     autoload :URL, 'cloudkeeper/utils/url'
+    autoload :Appliance, 'cloudkeeper/utils/appliance'
+    autoload :Date, 'cloudkeeper/utils/date'
   end
 end

data/lib/cloudkeeper/utils/appliance.rb

@@ -0,0 +1,60 @@
+module Cloudkeeper
+  module Utils
+    module Appliance
+      IMAGE_UPDATE_ATTRIBUTES = ['hv:version', 'sl:checksum:sha512', 'hv:size'].freeze
+
+      def log_expired(appliance, message)
+        logger.info "#{message} #{appliance.identifier.inspect}"
+      end
+
+      def clean_image_files(appliance)
+        return unless appliance && appliance.image
+
+        logger.debug "Cleaning downloaded image files for appliance #{appliance.identifier.inspect}"
+        appliance.image.image_files.each { |image_file| clean_image_file image_file.file }
+      rescue ::IOError => ex
+        logger.warn "Appliance cleanup error: #{ex.message}"
+      end
+
+      def clean_image_file(filename)
+        File.delete(filename) if File.exist?(filename)
+      end
+
+      def update_image?(image_list_appliance, backend_appliance)
+        image_list_attributes = image_list_appliance.attributes
+        backend_attributes = backend_appliance.attributes
+
+        IMAGE_UPDATE_ATTRIBUTES.reduce(false) { |red, elem| red || (image_list_attributes[elem] != backend_attributes[elem]) }
+      end
+
+      def update_metadata?(image_list_appliance, backend_appliance)
+        image_list_appliance.attributes != backend_appliance.attributes
+      end
+
+      def update_appliance(appliance)
+        modify_appliance :update_appliance, appliance
+      end
+
+      def prepare_image!(appliance)
+        image_file = Cloudkeeper::Managers::ImageManager.download_image(appliance.image.uri)
+        appliance.image.add_image_file image_file
+        return if acceptable_formats.include? image_file.format
+
+        convert_image! appliance, image_file
+      end
+
+      def convert_image!(appliance, image_file)
+        format = acceptable_formats.find { |acceptable_format| image_file.respond_to? "to_#{acceptable_format}".to_sym }
+        unless format
+          raise Cloudkeeper::Errors::Image::Format::NoRequiredFormatAvailableError,
+                "image #{image.inspect} cannot be converted to any acceptable format"
+        end
+
+        appliance.image.add_image_file image_file.send("to_#{format}".to_sym)
+      rescue Cloudkeeper::Errors::Image::Format::NoRequiredFormatAvailableError, Cloudkeeper::Errors::CommandExecutionError,
+             Cloudkeeper::Errors::ArgumentError, ::IOError => ex
+        raise Cloudkeeper::Errors::Image::ConversionError, ex
+      end
+    end
+  end
+end

data/lib/cloudkeeper/utils/date.rb

@@ -0,0 +1,11 @@
+module Cloudkeeper
+  module Utils
+    class Date
+      DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'.freeze
+
+      def self.parse(date)
+        date.blank? ? '' : Time.strptime(date, DATE_FORMAT)
+      end
+    end
+  end
+end

data/lib/cloudkeeper/version.rb

@@ -1,3 +1,3 @@
 module Cloudkeeper
-  VERSION = '1.5.1'.freeze
+  VERSION = '1.6.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudkeeper
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.6.0
 platform: ruby
 authors:
 - Michal Kimle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-15 00:00:00.000000000 Z
+date: 2017-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -25,61 +25,81 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.13'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: diffy
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: grpc-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 1.2.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 1.2.5
+- !ruby/object:Gem::Dependency
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.10'
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.48'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.48'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
-  name: rubocop-rspec
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: rspec-collection_matchers
   requirement: !ruby/object:Gem::Requirement
@@ -95,49 +115,49 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.48'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.48'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '1.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
-  name: vcr
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0.12'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -151,67 +171,67 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: diffy
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: grpc-tools
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
-    - - "<="
+        version: '4.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.2.5
-  type: :development
+        version: '6.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
-    - - "<="
+        version: '4.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.2.5
+        version: '6.0'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
-  name: yell
+  name: grpc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: mixlib-shellout
   requirement: !ruby/object:Gem::Requirement
@@ -226,26 +246,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
-- !ruby/object:Gem::Dependency
-  name: grpc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.1'
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 1.2.5
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.1'
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 1.2.5
 - !ruby/object:Gem::Dependency
   name: settingslogic
   requirement: !ruby/object:Gem::Requirement
@@ -261,41 +261,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: zaru
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.19'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: tilt
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: tilt
+  name: yell
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -309,19 +303,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: faraday
+  name: zaru
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.1'
 description: Synchronize cloud appliances between AppDB and cloud platforms
 email:
 - kimle.michal@gmail.com
@@ -406,7 +400,9 @@ files:
 - lib/cloudkeeper/nginx/templates/nginx.conf.erb
 - lib/cloudkeeper/settings.rb
 - lib/cloudkeeper/utils.rb
+- lib/cloudkeeper/utils/appliance.rb
 - lib/cloudkeeper/utils/checksum.rb
+- lib/cloudkeeper/utils/date.rb
 - lib/cloudkeeper/utils/hash.rb
 - lib/cloudkeeper/utils/url.rb
 - lib/cloudkeeper/version.rb
@@ -446,7 +442,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Synchronize cloud appliances between AppDB and cloud platforms