vmpooler 0.11.1 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52bdb481f0c4e9f3bfd690d4f85c906be9d15324b5c67d4f958fb64e290e8fd0
-  data.tar.gz: 941ee17c0f392d87a1691488a4479b607c05162ca4cb971929f71e2220692fec
+  metadata.gz: 3f6f3dafe8bbb4d65b5dc088ff36c24b12a3e6382aa2ec84f5a504e58e36eb6c
+  data.tar.gz: e488162ea027f566d44c762aa10095529d5610048bff7674e4695c16b37d61a1
 SHA512:
-  metadata.gz: 890cf253d1c32449ecc28d2c8ca143139fdda22a700fc4f90512cdfff15eab5a7e787cce4d57c89da517fd6f0ec589d25eb6cf5254b465894f92c83262af9058
-  data.tar.gz: da550116fa03211da3db4e03fd5fe662bc80da252a82980d462eaf1bda35404591cf0f63d148e485e5d922ba647afd1e0077c2edeca55d6797ce72cbd2f629a5
+  metadata.gz: 3ba72cb12cce3ab544437948751662434ce936014fdb68e63bbd6484bcceddaade2918b51852a6fd7d8171ea9a28d14135102cbe23cbae521fcbeb7f40ad6f7a
+  data.tar.gz: 484fd410cd7f96bde01bc928e3afc4cb5fb9eeebffdda8ebf615fd0414d1db10a294f28afdacc0e5cac3cf7448fc8c3a4636b50b2d02c4dc017034da6a0c4f41

data/bin/vmpooler

@@ -7,6 +7,8 @@ config = Vmpooler.config
 redis_host = config[:redis]['server']
 redis_port = config[:redis]['port']
 redis_password = config[:redis]['password']
+redis_connection_pool_size = config[:redis]['connection_pool_size']
+redis_connection_pool_timeout = config[:redis]['connection_pool_timeout']
 logger_file = config[:config]['logfile']
 
 metrics = Vmpooler.new_metrics(config)
@@ -36,7 +38,7 @@ if torun.include? 'manager'
     Vmpooler::PoolManager.new(
       config,
       Vmpooler.new_logger(logger_file),
-      Vmpooler.new_redis(redis_host, redis_port, redis_password),
+      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics),
       metrics
     ).execute!
   end

data/lib/vmpooler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Vmpooler
+  require 'concurrent'
   require 'date'
   require 'json'
   require 'net/ldap'
@@ -58,9 +59,14 @@ module Vmpooler
 
     # Set some configuration defaults
     parsed_config[:config]['task_limit'] = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
+    parsed_config[:config]['ondemand_clone_limit'] = string_to_int(ENV['ONDEMAND_CLONE_LIMIT']) || parsed_config[:config]['ondemand_clone_limit'] || 10
+    parsed_config[:config]['max_ondemand_instances_per_request'] = string_to_int(ENV['MAX_ONDEMAND_INSTANCES_PER_REQUEST']) || parsed_config[:config]['max_ondemand_instances_per_request'] || 10
     parsed_config[:config]['migration_limit'] = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
     parsed_config[:config]['vm_checktime'] = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
     parsed_config[:config]['vm_lifetime'] = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24
+    parsed_config[:config]['max_lifetime_upper_limit'] = string_to_int(ENV['MAX_LIFETIME_UPPER_LIMIT']) || parsed_config[:config]['max_lifetime_upper_limit']
+    parsed_config[:config]['ready_ttl'] = string_to_int(ENV['READY_TTL']) || parsed_config[:config]['ready_ttl'] || 60
+    parsed_config[:config]['ondemand_request_ttl'] = string_to_int(ENV['ONDEMAND_REQUEST_TTL']) || parsed_config[:config]['ondemand_request_ttl'] || 5
     parsed_config[:config]['prefix'] = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
 
     parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE']
@@ -72,7 +78,7 @@ module Vmpooler
     parsed_config[:config]['vm_lifetime_auth'] = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
     parsed_config[:config]['max_tries'] = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
     parsed_config[:config]['retry_factor'] = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
-    parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
+    parsed_config[:config]['create_folders'] = true?(ENV['CREATE_FOLDERS']) if ENV['CREATE_FOLDERS']
     parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
     set_linked_clone(parsed_config)
     parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
@@ -84,6 +90,8 @@ module Vmpooler
     parsed_config[:redis]['port'] = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT']
     parsed_config[:redis]['password'] = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
     parsed_config[:redis]['data_ttl'] = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
+    parsed_config[:redis]['connection_pool_size'] = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10
+    parsed_config[:redis]['connection_pool_timeout'] = string_to_int(ENV['REDIS_CONNECTION_POOL_TIMEOUT']) || parsed_config[:redis]['connection_pool_timeout'] || 5
 
     parsed_config[:statsd] = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
     parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
@@ -117,6 +125,7 @@ module Vmpooler
 
     parsed_config[:pools].each do |pool|
       parsed_config[:pool_names] << pool['name']
+      pool['ready_ttl'] ||= parsed_config[:config]['ready_ttl']
       if pool['alias']
         if pool['alias'].is_a?(Array)
           pool['alias'].each do |pool_alias|
@@ -154,6 +163,19 @@ module Vmpooler
     pools
   end
 
+  def self.redis_connection_pool(host, port, password, size, timeout, metrics)
+    Vmpooler::PoolManager::GenericConnectionPool.new(
+      metrics: metrics,
+      metric_prefix: 'redis_connection_pool',
+      size: size,
+      timeout: timeout
+    ) do
+      connection = Concurrent::Hash.new
+      redis = new_redis(host, port, password)
+      connection['connection'] = redis
+    end
+  end
+
   def self.new_redis(host = 'localhost', port = nil, password = nil)
     Redis.new(host: host, port: port, password: password)
   end

data/lib/vmpooler/api/helpers.rb

@@ -84,29 +84,29 @@ module Vmpooler
         when 'ldap'
           ldap_base = auth[:ldap]['base']
           ldap_port = auth[:ldap]['port'] || 389
+          ldap_user_obj = auth[:ldap]['user_object']
+          ldap_host = auth[:ldap]['host']
 
-          if ldap_base.is_a? Array
-            ldap_base.each do |search_base|
+          unless ldap_base.is_a? Array
+            ldap_base = ldap_base.split
+          end
+
+          unless ldap_user_obj.is_a? Array
+            ldap_user_obj = ldap_user_obj.split
+          end
+
+          ldap_base.each do |search_base|
+            ldap_user_obj.each do |search_user_obj|
               result = authenticate_ldap(
                 ldap_port,
-                auth[:ldap]['host'],
-                auth[:ldap]['user_object'],
+                ldap_host,
+                search_user_obj,
                 search_base,
                 username_str,
                 password_str
               )
-              return true if result == true
+              return true if result
             end
-          else
-            result = authenticate_ldap(
-              ldap_port,
-              auth[:ldap]['host'],
-              auth[:ldap]['user_object'],
-              ldap_base,
-              username_str,
-              password_str
-            )
-            return result
           end
 
           return false
@@ -238,7 +238,7 @@ module Vmpooler
         queue[:running]   = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
         queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
 
-        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
+        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i
         queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
         queue[:booting] = 0 if queue[:booting] < 0
         queue[:total]   = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i

data/lib/vmpooler/api/v1.rb

@@ -42,6 +42,68 @@ module Vmpooler
       Vmpooler::API.settings.checkoutlock
     end
 
+    def get_template_aliases(template)
+      result = []
+      aliases = Vmpooler::API.settings.config[:alias]
+      if aliases
+        result += aliases[template] if aliases[template].is_a?(Array)
+        template_backends << aliases[template] if aliases[template].is_a?(String)
+      end
+      result
+    end
+
+    def get_pool_weights(template_backends)
+      pool_index = pool_index(pools)
+      weighted_pools = {}
+      template_backends.each do |t|
+        next unless pool_index.key? t
+
+        index = pool_index[t]
+        clone_target = pools[index]['clone_target'] || config['clone_target']
+        next unless config.key?('backend_weight')
+
+        weight = config['backend_weight'][clone_target]
+        if weight
+          weighted_pools[t] = weight
+        end
+      end
+      weighted_pools
+    end
+
+    def count_selection(selection)
+      result = {}
+      selection.uniq.each do |poolname|
+        result[poolname] = selection.count(poolname)
+      end
+      result
+    end
+
+    def evaluate_template_aliases(template, count)
+      template_backends = []
+      template_backends << template if backend.sismember('vmpooler__pools', template)
+      selection = []
+      aliases = get_template_aliases(template)
+      if aliases
+        template_backends += aliases
+        weighted_pools = get_pool_weights(template_backends)
+
+        pickup = Pickup.new(weighted_pools) if weighted_pools.count == template_backends.count
+        count.to_i.times do
+          if pickup
+            selection << pickup.pick
+          else
+            selection << template_backends.sample
+          end
+        end
+      else
+        count.to_i.times do
+          selection << template
+        end
+      end
+
+      count_selection(selection)
+    end
+
     def fetch_single_vm(template)
       template_backends = [template]
       aliases = Vmpooler::API.settings.config[:alias]
@@ -83,8 +145,13 @@ module Vmpooler
           vms.reverse.each do |vm|
             ready = vm_ready?(vm, config['domain'])
             if ready
-              backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
-              return [vm, template_backend, template]
+              smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
+              if smoved
+                return [vm, template_backend, template]
+              else
+                metrics.increment("checkout.smove.failed.#{template_backend}")
+                return [nil, nil, nil]
+              end
             else
               backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
               metrics.increment("checkout.nonresponsive.#{template_backend}")
@@ -240,11 +307,9 @@ module Vmpooler
       pool_index = pool_index(pools)
       template_configs = backend.hgetall('vmpooler__config__template')
       template_configs&.each do |poolname, template|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['template'] == template
-              pools[pool_index[poolname]]['template'] = template
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['template'] = template
       end
     end
 
@@ -252,11 +317,9 @@ module Vmpooler
       pool_index = pool_index(pools)
       poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
       poolsize_configs&.each do |poolname, size|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['size'] == size.to_i
-              pools[pool_index[poolname]]['size'] == size.to_i
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['size'] = size.to_i
       end
     end
 
@@ -264,12 +327,67 @@ module Vmpooler
       pool_index = pool_index(pools)
       clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
       clone_target_configs&.each do |poolname, clone_target|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['clone_target'] == clone_target
-              pools[pool_index[poolname]]['clone_target'] == clone_target
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['clone_target'] = clone_target
+      end
+    end
+
+    def too_many_requested?(payload)
+      payload&.each do |_poolname, count|
+        next unless count.to_i > config['max_ondemand_instances_per_request']
+
+        return true
+      end
+      false
+    end
+
+    def generate_ondemand_request(payload)
+      result = { 'ok': false }
+
+      requested_instances = payload.reject { |k, _v| k == 'request_id' }
+      if too_many_requested?(requested_instances)
+        result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
+        status 403
+        return result
+      end
+
+      score = Time.now.to_i
+      request_id = payload['request_id']
+      request_id ||= generate_request_id
+      result['request_id'] = request_id
+
+      if backend.exists("vmpooler__odrequest__#{request_id}")
+        result['message'] = "request_id '#{request_id}' has already been created"
+        status 409
+        return result
+      end
+
+      status 201
+
+      platforms_with_aliases = []
+      requested_instances.each do |poolname, count|
+        selection = evaluate_template_aliases(poolname, count)
+        selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
+      end
+      platforms_string = platforms_with_aliases.join(',')
+
+      return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
+
+      backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+      if Vmpooler::API.settings.config[:auth] and has_token?
+        backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
+        backend.hset("vmpooler__odrequest__#{request_id}", 'token:user',
+                     backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user'))
       end
+
+      result['domain'] = config['domain'] if config['domain']
+      result[:ok] = true
+      result
+    end
+
+    def generate_request_id
+      SecureRandom.uuid
     end
 
     get '/' do
@@ -390,7 +508,7 @@ module Vmpooler
           end
 
           # for backwards compatibility, include separate "empty" stats in "status" block
-          if ready == 0
+          if ready == 0 && max != 0
             result[:status][:empty] ||= []
             result[:status][:empty].push(pool['name'])
 
@@ -684,6 +802,88 @@ module Vmpooler
       JSON.pretty_generate(result)
     end
 
+    post "#{api_prefix}/ondemandvm/?" do
+      content_type :json
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      result = { 'ok' => false }
+
+      begin
+        payload = JSON.parse(request.body.read)
+
+        if payload
+          invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+          if invalid.empty?
+            result = generate_ondemand_request(payload)
+          else
+            result[:bad_templates] = invalid
+            invalid.each do |bad_template|
+              metrics.increment('ondemandrequest.invalid.' + bad_template)
+            end
+            status 404
+          end
+        else
+          metrics.increment('ondemandrequest.invalid.unknown')
+          status 404
+        end
+      rescue JSON::ParserError
+        status 400
+        result = {
+          'ok' => false,
+          'message' => 'JSON payload could not be parsed'
+        }
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    post "#{api_prefix}/ondemandvm/:template/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      payload = extract_templates_from_query_params(params[:template])
+
+      if payload
+        invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+        if invalid.empty?
+          result = generate_ondemand_request(payload)
+        else
+          result[:bad_templates] = invalid
+          invalid.each do |bad_template|
+            metrics.increment('ondemandrequest.invalid.' + bad_template)
+          end
+          status 404
+        end
+      else
+        metrics.increment('ondemandrequest.invalid.unknown')
+        status 404
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    get "#{api_prefix}/ondemandvm/:requestid/?" do
+      content_type :json
+
+      status 404
+      result = check_ondemand_request(params[:requestid])
+
+      JSON.pretty_generate(result)
+    end
+
+    delete "#{api_prefix}/ondemandvm/:requestid/?" do
+      content_type :json
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      status 404
+      result = delete_ondemand_request(params[:requestid])
+
+      JSON.pretty_generate(result)
+    end
+
     post "#{api_prefix}/vm/?" do
       content_type :json
       result = { 'ok' => false }
@@ -759,6 +959,78 @@ module Vmpooler
       invalid
     end
 
+    def check_ondemand_request(request_id)
+      result = { 'ok' => false }
+      request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
+      if request_hash.empty?
+        result['message'] = "no request found for request_id '#{request_id}'"
+        return result
+      end
+
+      result['request_id'] = request_id
+      result['ready'] = false
+      result['ok'] = true
+      status 202
+
+      if request_hash['status'] == 'ready'
+        result['ready'] = true
+        platform_parts = request_hash['requested'].split(',')
+        platform_parts.each do |platform|
+          pool_alias, pool, _count = platform.split(':')
+          instances = backend.smembers("vmpooler__#{request_id}__#{pool_alias}__#{pool}")
+          result[pool_alias] = { 'hostname': instances }
+        end
+        result['domain'] = config['domain'] if config['domain']
+        status 200
+      elsif request_hash['status'] == 'failed'
+        result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
+        status 200
+      elsif request_hash['status'] == 'deleted'
+        result['message'] = 'The request has been deleted'
+        status 200
+      else
+        platform_parts = request_hash['requested'].split(',')
+        platform_parts.each do |platform|
+          pool_alias, pool, count = platform.split(':')
+          instance_count = backend.scard("vmpooler__#{request_id}__#{pool_alias}__#{pool}")
+          result[pool_alias] = {
+            'ready': instance_count.to_s,
+            'pending': (count.to_i - instance_count.to_i).to_s
+          }
+        end
+      end
+
+      result
+    end
+
+    def delete_ondemand_request(request_id)
+      result = { 'ok' => false }
+
+      platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
+      unless platforms
+        result['message'] = "no request found for request_id '#{request_id}'"
+        return result
+      end
+
+      if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
+        result['message'] = 'the request has already been deleted'
+      else
+        backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
+
+        platforms.split(',').each do |platform|
+          pool_alias, pool, _count = platform.split(':')
+          backend.smembers("vmpooler__#{request_id}__#{pool_alias}__#{pool}")&.each do |vm|
+            backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
+          end
+          backend.del("vmpooler__#{request_id}__#{pool_alias}__#{pool}")
+        end
+        backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
+      end
+      status 200
+      result['ok'] = true
+      result
+    end
+
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
@@ -874,6 +1146,8 @@ module Vmpooler
 
           status 200
           result['ok'] = true
+        else
+          metrics.increment('delete.srem.failed')
         end
       end
 
@@ -909,13 +1183,6 @@ module Vmpooler
                 max_lifetime_upper_limit = max_lifetime_upper_limit.to_i
                 if arg.to_i >= max_lifetime_upper_limit
                   failure.push("You provided a lifetime (#{arg}) that exceeds the configured maximum of #{max_lifetime_upper_limit}.")
-                else
-                  # also make sure we do not extend past max_lifetime_upper_limit
-                  rdata = backend.hgetall('vmpooler__vm__' + params[:hostname])
-                  running = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
-                  unless running + arg.to_i < max_lifetime_upper_limit
-                    failure.push("You provided a lifetime (#{arg}) that will extend the current lifetime past the configured maximum of #{max_lifetime_upper_limit}.")
-                  end
                 end
               end
 
@@ -923,6 +1190,7 @@ module Vmpooler
               unless arg.to_i > 0
                 failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.")
               end
+
             when 'tags'
               unless arg.is_a?(Hash)
                 failure.push("You provided tags (#{arg}) as something other than a hash.")
@@ -1047,7 +1315,7 @@ module Vmpooler
             invalid.each do |bad_template|
               metrics.increment("config.invalid.#{bad_template}")
             end
-            result[:bad_templates] = invalid
+            result[:not_configured] = invalid
             status 400
           end
         else