cloudkeeper 1.3.1 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a7b923d81dc5b5f94d50e099e7775f7994d0e8f
-  data.tar.gz: 551167065e752bbc83ff8c0d4402903a5f70f418
+  metadata.gz: 627d7ad80b5656af87fef1d88bdaa35fae85cda3
+  data.tar.gz: c4f5fa24ab19bf98e727e9a36be4cf1581b12005
 SHA512:
-  metadata.gz: 17d9205bdac272454f9936230096e44058568f94998ff692ca6c6d3a3d5f1c3eb7c27cc785525bc19b0245e36e0bdded83f24dcc07c4551c55110d1d3177815a
-  data.tar.gz: 9d8ea1adf2e5dccfaf5f55b673d4a494d80269d4536fd64e7c72d79851f900f8abb1b1593b356d72a5d9d442d249162c4ce065d73ca396d8a6a30a50427a6ad9
+  metadata.gz: feaefd01329102a82e8739c481c828f4b18df12b572bed7b4f2cde0bf4f864b1cdc5a6215db65a4434530bad569cede8ba04e6f34f5d6e4ad04d0e3394c3d8c6
+  data.tar.gz: 7a6c49eb1bbf1db0ffd46f855db2693cd02b292d03dfd5996429b22631bfec0db367d349fd7b5eb60af30ac91d9815ef9d1e2c483c3b562f64b7c69e42fc0448

data/.rubocop.yml

@@ -28,6 +28,11 @@ Metrics/ClassLength:
   Exclude:
     - 'lib/cloudkeeper/cli.rb'
 
+Metrics/ClassLength:
+  Max: 125
+  Include:
+    - 'lib/cloudkeeper/managers/appliance_manager.rb'
+
 RSpec/MultipleExpectations:
   Enabled: false
 

data/README.md

@@ -71,50 +71,53 @@ cloudkeeper is run with executable `cloudkeeper`. For further assistance run `cl
 $ cloudkeeper help sync
 
 Usage:
-  cloudkeeper sync --backend-endpoint=BACKEND-ENDPOINT --formats=one two three --image-dir=IMAGE-DIR --image-lists=one two three --qemu-img-binary=QEMU-IMG-BINARY
+  cloudkeeper sync --backend-endpoint=BACKEND-ENDPOINT --external-tools-execution-timeout=N --formats=one two three --image-dir=IMAGE-DIR --image-lists=one two three --qemu-img-binary=QEMU-IMG-BINARY
 
 Options:
-  --image-lists=one two three                      # List of image lists to sync against
-  [--ca-dir=CA-DIR]                                # CA directory
-                                                   # Default: /etc/grid-security/certificates/
-  [--authentication], [--no-authentication]        # Client <-> server authentication
-  [--certificate=CERTIFICATE]                      # Core's host certificate
-                                                   # Default: /etc/grid-security/hostcert.pem
-  [--key=KEY]                                      # Core's host key
-                                                   # Default: /etc/grid-security/hostkey.pem
-  --image-dir=IMAGE-DIR                            # Directory to store images to
-                                                   # Default: /var/spool/cloudkeeper/images/
-  --qemu-img-binary=QEMU-IMG-BINARY                # Path to qemu-img binary (image conversion)
-                                                   # Default: /usr/bin/qemu-img
-  [--nginx-binary=NGINX-BINARY]                    # Path to nginx binary (HTTP server)
-                                                   # Default: /usr/bin/nginx
-  [--remote-mode], [--no-remote-mode]              # Remote mode starts HTTP server (NGINX) and serves images to backend via HTTP
-  [--nginx-error-log-file=NGINX-ERROR-LOG-FILE]    # NGINX error log file
-                                                   # Default: /var/log/cloudkeeper/nginx-error.log
-  [--nginx-access-log-file=NGINX-ACCESS-LOG-FILE]  # NGINX access log file
-                                                   # Default: /var/log/cloudkeeper/nginx-access.log
-  [--nginx-pid-file=NGINX-PID-FILE]                # NGINX pid file
-                                                   # Default: /var/run/cloudkeeper/nginx.pid
-  [--nginx-ip-address=NGINX-IP-ADDRESS]            # IP address NGINX can listen on
-                                                   # Default: 127.0.0.1
-  [--nginx-min-port=N]                             # Minimal port NGINX can listen on
-                                                   # Default: 7300
-  [--nginx-max-port=N]                             # Maximal port NGINX can listen on
-                                                   # Default: 7400
-  --backend-endpoint=BACKEND-ENDPOINT              # Backend's gRPC endpoint
-                                                   # Default: 127.0.0.1:50051
-  [--backend-certificate=BACKEND-CERTIFICATE]      # Backend's certificate
-                                                   # Default: /etc/grid-security/backendcert.pem
-  --formats=one two three                          # List of acceptable formats images can be converted to
-                                                   # Default: ["qcow2"]
+  --image-lists=one two three                        # List of image lists to sync against
+  [--ca-dir=CA-DIR]                                  # CA directory
+                                                     # Default: /etc/grid-security/certificates/
+  [--authentication], [--no-authentication]          # Client <-> server authentication
+  [--certificate=CERTIFICATE]                        # Core's host certificate
+                                                     # Default: /etc/grid-security/hostcert.pem
+  [--key=KEY]                                        # Core's host key
+                                                     # Default: /etc/grid-security/hostkey.pem
+  --image-dir=IMAGE-DIR                              # Directory to store images to
+                                                     # Default: /var/spool/cloudkeeper/images/
+  --qemu-img-binary=QEMU-IMG-BINARY                  # Path to qemu-img binary (image conversion)
+                                                     # Default: /usr/bin/qemu-img
+  [--nginx-binary=NGINX-BINARY]                      # Path to nginx binary (HTTP server)
+                                                     # Default: /usr/bin/nginx
+  --external-tools-execution-timeout=N               # Timeout for execution of external tools in seconds
+                                                     # Default: 600
+  [--remote-mode], [--no-remote-mode]                # Remote mode starts HTTP server (NGINX) and serves images to backend via HTTP
+  [--nginx-error-log-file=NGINX-ERROR-LOG-FILE]      # NGINX error log file
+                                                     # Default: /var/log/cloudkeeper/nginx-error.log
+  [--nginx-access-log-file=NGINX-ACCESS-LOG-FILE]    # NGINX access log file
+                                                     # Default: /var/log/cloudkeeper/nginx-access.log
+  [--nginx-pid-file=NGINX-PID-FILE]                  # NGINX pid file
+                                                     # Default: /var/run/cloudkeeper/nginx.pid
+  [--nginx-ip-address=NGINX-IP-ADDRESS]              # IP address NGINX can listen on
+                                                     # Default: 127.0.0.1
+  [--nginx-port=N]                                   # Port NGINX can listen on
+                                                     # Default: 50505
+  [--nginx-proxy-ip-address=NGINX-PROXY-IP-ADDRESS]  # Proxy IP address
+  [--nginx-proxy-port=N]                             # Proxy port
+  [--nginx-proxy-ssl], [--no-nginx-proxy-ssl]        # Whether proxy will use SSL connection
+  --backend-endpoint=BACKEND-ENDPOINT                # Backend's gRPC endpoint
+                                                     # Default: 127.0.0.1:50051
+  [--backend-certificate=BACKEND-CERTIFICATE]        # Backend's certificate
+                                                     # Default: /etc/grid-security/backendcert.pem
+  --formats=one two three                            # List of acceptable formats images can be converted to
+                                                     # Default: ["qcow2"]
   --logging-level=LOGGING-LEVEL
-                                                   # Default: ERROR
-                                                   # Possible values: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN
-  [--logging-file=LOGGING-FILE]                    # File to write logs to
-                                                   # Default: /var/log/cloudkeeper/cloudkeeper.log
-  --lock-file=LOCK-FILE                            # File used to ensure only one running instance of cloudkeeper
-                                                   # Default: /var/lock/cloudkeeper/cloudkeeper.lock
-  [--debug], [--no-debug]                          # Runs cloudkeeper in debug mode
+                                                     # Default: ERROR
+                                                     # Possible values: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN
+  [--logging-file=LOGGING-FILE]                      # File to write logs to
+                                                     # Default: /var/log/cloudkeeper/cloudkeeper.log
+  --lock-file=LOCK-FILE                              # File used to ensure only one running instance of cloudkeeper
+                                                     # Default: /var/lock/cloudkeeper/cloudkeeper.lock
+  [--debug], [--no-debug]                            # Runs cloudkeeper in debug mode
 ```
 
 ## Contributing

data/cloudkeeper.gemspec

@@ -42,12 +42,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'vcr', '~> 3.0'
   spec.add_development_dependency 'webmock', '~> 3.0'
   spec.add_development_dependency 'diffy', '~> 3.1'
-  spec.add_development_dependency 'grpc-tools', '~> 1.0'
+  spec.add_development_dependency 'grpc-tools', '>= 1.1', '<= 1.2.5'
 
   spec.add_runtime_dependency 'thor', '~> 0.19'
   spec.add_runtime_dependency 'yell', '~> 2.0'
   spec.add_runtime_dependency 'mixlib-shellout', '~> 2.2'
-  spec.add_runtime_dependency 'grpc', '~> 1.0'
+  spec.add_runtime_dependency 'grpc', '>= 1.1', '<= 1.2.5'
   spec.add_runtime_dependency 'settingslogic', '~> 2.0'
   spec.add_runtime_dependency 'zaru', '~> 0.1'
   spec.add_runtime_dependency 'activesupport', '>= 4.0', '< 6.0'

data/config/cloudkeeper.yml

@@ -5,9 +5,11 @@ cloudkeeper:
   certificate: /etc/grid-security/hostcert.pem # Core's host certificate
   key: /etc/grid-security/hostkey.pem # Core's host key
   image-dir: /var/spool/cloudkeeper/images/ # Directory to store images to
-  binaries:
-    qemu-img: /usr/bin/qemu-img # qemu-img binary (image conversion) location
-    nginx: /usr/bin/nginx # nginx binary (HTTP server) location
+  external-tools:
+    binaries:
+      qemu-img: /usr/bin/qemu-img # qemu-img binary (image conversion) location
+      nginx: /usr/bin/nginx # nginx binary (HTTP server) location
+    execution-timeout: 600 # timeout for execution of external tools in seconds
   remote-mode: false # Remote mode starts HTTP server (NGINX) and serves images to backend via HTTP
   nginx:
     runtime-dir: /var/run/cloudkeeper/ # Runtime directory for NGINX

data/lib/cloudkeeper/backend_connector.rb

@@ -20,22 +20,23 @@ module Cloudkeeper
     end
 
     def add_appliance(appliance)
-      logger.debug "'add_appliance' gRPC method call"
+      logger.debug "'add_appliance' gRPC method call (appliance.identifier: #{appliance.identifier})"
       manage_appliance appliance, :add_appliance
     end
 
     def update_appliance(appliance)
-      logger.debug "'update_appliance' gRPC method call"
+      logger.debug "'update_appliance' gRPC method call (appliance.identifier: #{appliance.identifier})"
       manage_appliance appliance, :update_appliance
     end
 
     def remove_appliance(appliance)
-      logger.debug "'remove_appliance' gRPC method call"
+      logger.debug "'remove_appliance' gRPC method call (appliance.identifier: #{appliance.identifier})"
+      appliance.image = nil
       manage_appliance appliance, :remove_appliance
     end
 
     def remove_image_list(image_list_identifier)
-      logger.debug "'remove_image_list' gRPC method call"
+      logger.debug "'remove_image_list' gRPC method call (image_list_identifier: #{image_list_identifier})"
       handle_errors grpc_client.remove_image_list(
         CloudkeeperGrpc::ImageListIdentifier.new(image_list_identifier: image_list_identifier),
         return_op: true

data/lib/cloudkeeper/cli.rb

@@ -53,13 +53,18 @@ module Cloudkeeper
                   desc: 'Directory to store images to'
     method_option :'qemu-img-binary',
                   required: true,
-                  default: Cloudkeeper::Settings['binaries']['qemu-img'],
+                  default: Cloudkeeper::Settings['external-tools']['binaries']['qemu-img'],
                   type: :string,
                   desc: 'Path to qemu-img binary (image conversion)'
     method_option :'nginx-binary',
-                  default: Cloudkeeper::Settings['binaries']['nginx'],
+                  default: Cloudkeeper::Settings['external-tools']['binaries']['nginx'],
                   type: :string,
                   desc: 'Path to nginx binary (HTTP server)'
+    method_option :'external-tools-execution-timeout',
+                  required: true,
+                  default: Cloudkeeper::Settings['external-tools']['execution-timeout'],
+                  type: :numeric,
+                  desc: 'Timeout for execution of external tools in seconds'
     method_option :'remote-mode',
                   default: Cloudkeeper::Settings['remote-mode'],
                   type: :boolean,

data/lib/cloudkeeper/command_executioner.rb

@@ -2,7 +2,7 @@ module Cloudkeeper
   class CommandExecutioner
     class << self
       def execute(*args)
-        command = Mixlib::ShellOut.new(*args)
+        command = Mixlib::ShellOut.new(*args, timeout: Cloudkeeper::Settings[:'external-tools-execution-timeout'])
         logger.debug "Executing command: #{command.command.inspect}"
         command.run_command
 

data/lib/cloudkeeper/entities/appliance.rb

@@ -54,8 +54,8 @@ module Cloudkeeper
                                     appliance_hash[:'dc:title'],
                                     appliance_hash[:'dc:description'],
                                     appliance_hash[:'ad:group'],
-                                    appliance_hash[:'ad:ram_recommended'],
-                                    appliance_hash[:'ad:core_recommended'],
+                                    appliance_hash[:'hv:ram_minimum'],
+                                    appliance_hash[:'hv:core_minimum'],
                                     appliance_hash[:'hv:version'],
                                     appliance_hash[:'sl:arch']
 

data/lib/cloudkeeper/entities/convertables/convertable.rb

@@ -6,12 +6,6 @@ module Cloudkeeper
       module Convertable
         CONVERT_OUTPUT_FORMATS = %i[raw qcow2 vmdk vdi].freeze
 
-        def self.convert_output_formats
-          CONVERT_OUTPUT_FORMATS
-        end
-
-        FORMAT_REGEX = /^to_(?<format>#{convert_output_formats.join('|')})$/
-
         def self.included(base)
           raise Cloudkeeper::Errors::Convertables::ConvertabilityError, "#{base.inspect} cannot become a convertable" \
             unless base.method_defined?(:file) && base.method_defined?(:format)
@@ -19,27 +13,40 @@ module Cloudkeeper
           super
         end
 
+        def convert_output_formats
+          CONVERT_OUTPUT_FORMATS
+        end
+
+        def format_regex
+          /^to_(?<format>#{convert_output_formats.join('|')})$/
+        end
+
         def method_missing(method, *arguments, &block)
-          result = method.to_s.match(FORMAT_REGEX)
-          return convert(result[:format]) if result && result[:format]
+          result = method.to_s.match(format_regex)
+          if result && result[:format]
+            return self if format.to_sym == result[:format].to_sym
+            return convert result[:format]
+          end
 
           super
         end
 
         def respond_to_missing?(method, *)
-          method =~ FORMAT_REGEX || super
+          method =~ format_regex || super
         end
 
         private
 
         def convert(output_format)
           logger.debug "Converting file #{file.inspect} from #{format.inspect} to #{output_format.inspect}"
-          return self if output_format.to_sym == format.to_sym
 
           converted_file = File.join(File.dirname(file), "#{File.basename(file, '.*')}.#{output_format}")
           run_convert_command(output_format, converted_file)
 
           image_file converted_file, output_format
+        rescue Cloudkeeper::Errors::CommandExecutionError => ex
+          delete_if_exists converted_file
+          raise ex
         end
 
         def run_convert_command(output_format, converted_file)
@@ -57,6 +64,10 @@ module Cloudkeeper
           Cloudkeeper::Entities::ImageFile.new converted_file, output_format.to_sym,
                                                Cloudkeeper::Utils::Checksum.compute(converted_file), false
         end
+
+        def delete_if_exists(file)
+          File.delete(file) if File.exist?(file.to_s)
+        end
       end
     end
   end

data/lib/cloudkeeper/entities/convertables/ova.rb

@@ -2,11 +2,7 @@ module Cloudkeeper
   module Entities
     module Convertables
       module Ova
-        CONVERT_OUTPUT_FORMATS = %i[raw qcow2 vdi].freeze
-
-        def self.convert_output_formats
-          CONVERT_OUTPUT_FORMATS
-        end
+        CONVERT_OUTPUT_FORMATS = %i[raw qcow2 vdi ova].freeze
 
         def self.extended(base)
           raise Cloudkeeper::Errors::Convertables::ConvertabilityError, "#{base.inspect} cannot become OVA convertable" \
@@ -15,12 +11,13 @@ module Cloudkeeper
           super
         end
 
-        def to_vmdk
-          image_file(extract_disk, :vmdk)
+        def convert_output_formats
+          CONVERT_OUTPUT_FORMATS
         end
 
-        def to_ova
-          self
+        def to_vmdk
+          logger.debug "Converting file #{file.inspect} from #{format.inspect} to vmdk"
+          image_file(extract_disk, :vmdk)
         end
 
         private
@@ -37,8 +34,12 @@ module Cloudkeeper
         def extract_disk
           archived_disk = disk_file
           disk_directory = File.dirname(file)
+          extracted_file = File.join(disk_directory, archived_disk)
           Cloudkeeper::CommandExecutioner.execute('tar', '-x', '-f', file, '-C', disk_directory, archived_disk)
-          File.join(disk_directory, archived_disk)
+          extracted_file
+        rescue Cloudkeeper::Errors::CommandExecutionError => ex
+          delete_if_exists extracted_file
+          raise ex
         end
 
         def archive_files

data/lib/cloudkeeper/managers/appliance_manager.rb

@@ -39,6 +39,7 @@ module Cloudkeeper
       def sync_new_image_lists(backend_image_lists)
         logger.debug 'Registering appliances from new image lists...'
         add_list = image_list_manager.image_lists.keys - backend_image_lists
+        logger.debug "Image lists to register: #{add_list.inspect}"
         add_list.each do |image_list_identifier|
           image_list_manager.image_lists[image_list_identifier].appliances.each_value { |appliance| add_appliance appliance }
         end
@@ -47,6 +48,7 @@ module Cloudkeeper
       def sync_old_image_lists(backend_image_lists)
         logger.debug 'Synchronizing registered appliances...'
         sync_list = image_list_manager.image_lists.keys & backend_image_lists
+        logger.debug "Image lists to synchronize: #{sync_list.inspect}"
         sync_list.each { |image_list_identifier| sync_image_list image_list_identifier }
       end
 
@@ -63,18 +65,21 @@ module Cloudkeeper
       def remove_appliances(backend_appliances, image_list_appliances)
         logger.debug 'Removing previously registered appliances...'
         remove_list = backend_appliances.keys - image_list_appliances.keys
-        remove_list.each { |appliance_identifier| backend_connector.remove_appliance image_list_appliances[appliance_identifier] }
+        logger.debug "Appliances to remove: #{remove_list.inspect}"
+        remove_list.each { |appliance_identifier| backend_connector.remove_appliance backend_appliances[appliance_identifier] }
       end
 
       def add_appliances(backend_appliances, image_list_appliances)
         logger.debug 'Registering new appliances...'
         add_list = image_list_appliances.keys - backend_appliances.keys
+        logger.debug "Appliances to register: #{add_list.inspect}"
         add_list.each { |appliance_identifier| add_appliance image_list_appliances[appliance_identifier] }
       end
 
       def update_appliances(backend_appliances, image_list_appliances)
         logger.debug 'Updating appliances...'
         update_list = backend_appliances.keys & image_list_appliances.keys
+        logger.debug "Appliances for potential update: #{update_list.inspect}"
         update_list.each do |appliance_identifier|
           image_list_appliance = image_list_appliances[appliance_identifier]
           backend_appliance = backend_appliances[appliance_identifier]

data/lib/cloudkeeper/version.rb

@@ -1,3 +1,3 @@
 module Cloudkeeper
-  VERSION = '1.3.1'.freeze
+  VERSION = '1.4.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudkeeper
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.1
 platform: ruby
 authors:
 - Michal Kimle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-26 00:00:00.000000000 Z
+date: 2017-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -168,16 +168,22 @@ dependencies:
   name: grpc-tools
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.2.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.2.5
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -224,16 +230,22 @@ dependencies:
   name: grpc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.2.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.2.5
 - !ruby/object:Gem::Dependency
   name: settingslogic
   requirement: !ruby/object:Gem::Requirement