vmpooler 0.11.3 → 0.13.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ea79e2b20c34ba8ec261bcec65ed74ad0d214fdaaf605b440bfeb6d870016dc
-  data.tar.gz: 0c880b0c31ab2100aafaf3d6f6b83f50aa723fc44a0dded1a0bfdd0fa6bbc87f
+  metadata.gz: fe0ef9bcf3e665cc952fce0975a4381883fca200ef93a883e54c7db7e1fc10e7
+  data.tar.gz: d44b61c41a57f940cf0df4bf07aabed10b1a72e91cae5b2105f092a7d98d6d05
 SHA512:
-  metadata.gz: 68c5ce2a6a5111824a63c8610c4fd533ac0b538bb29b53064bd09bdd6fbe2b610f3464f5e2270e9de3afbfc1ee6ba0111beed1659e09431c12ebb5124eb305d4
-  data.tar.gz: 8a69ccbb59c189f8757597bcda0c5ccfe496685582d49992fa604a4fa047c484c424345bff7db60c2830891e48ee8155b4bd0afdc586c7860161f6a8f58da9b8
+  metadata.gz: e0e7bbec11ceecc5fdee0fe15b3d37e290a6c884a708a6a5acb2e790d54fb1f1e24ec6966973f02bbecd204a29497e9c17ae3b8be26a0d3f15cec508a0d64532
+  data.tar.gz: 7302cf41676a65ff5d0bb4108a8f5ac03a96a92e3cd0ec56deee2f2a60d6e73f09eee569ef0f4140e3c27e81a16c3e2d24b951764a772bdeaa082a96d8bde1fe

data/bin/vmpooler

@@ -7,6 +7,8 @@ config = Vmpooler.config
 redis_host = config[:redis]['server']
 redis_port = config[:redis]['port']
 redis_password = config[:redis]['password']
+redis_connection_pool_size = config[:redis]['connection_pool_size']
+redis_connection_pool_timeout = config[:redis]['connection_pool_timeout']
 logger_file = config[:config]['logfile']
 
 metrics = Vmpooler.new_metrics(config)
@@ -36,7 +38,7 @@ if torun.include? 'manager'
     Vmpooler::PoolManager.new(
       config,
       Vmpooler.new_logger(logger_file),
-      Vmpooler.new_redis(redis_host, redis_port, redis_password),
+      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics),
       metrics
     ).execute!
   end

data/lib/vmpooler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Vmpooler
+  require 'concurrent'
   require 'date'
   require 'json'
   require 'net/ldap'
@@ -58,9 +59,14 @@ module Vmpooler
 
     # Set some configuration defaults
     parsed_config[:config]['task_limit'] = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
+    parsed_config[:config]['ondemand_clone_limit'] = string_to_int(ENV['ONDEMAND_CLONE_LIMIT']) || parsed_config[:config]['ondemand_clone_limit'] || 10
+    parsed_config[:config]['max_ondemand_instances_per_request'] = string_to_int(ENV['MAX_ONDEMAND_INSTANCES_PER_REQUEST']) || parsed_config[:config]['max_ondemand_instances_per_request'] || 10
     parsed_config[:config]['migration_limit'] = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
     parsed_config[:config]['vm_checktime'] = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
     parsed_config[:config]['vm_lifetime'] = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24
+    parsed_config[:config]['max_lifetime_upper_limit'] = string_to_int(ENV['MAX_LIFETIME_UPPER_LIMIT']) || parsed_config[:config]['max_lifetime_upper_limit']
+    parsed_config[:config]['ready_ttl'] = string_to_int(ENV['READY_TTL']) || parsed_config[:config]['ready_ttl'] || 60
+    parsed_config[:config]['ondemand_request_ttl'] = string_to_int(ENV['ONDEMAND_REQUEST_TTL']) || parsed_config[:config]['ondemand_request_ttl'] || 5
     parsed_config[:config]['prefix'] = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
 
     parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE']
@@ -72,7 +78,7 @@ module Vmpooler
     parsed_config[:config]['vm_lifetime_auth'] = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
     parsed_config[:config]['max_tries'] = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
     parsed_config[:config]['retry_factor'] = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
-    parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
+    parsed_config[:config]['create_folders'] = true?(ENV['CREATE_FOLDERS']) if ENV['CREATE_FOLDERS']
     parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
     set_linked_clone(parsed_config)
     parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
@@ -84,6 +90,8 @@ module Vmpooler
     parsed_config[:redis]['port'] = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT']
     parsed_config[:redis]['password'] = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
     parsed_config[:redis]['data_ttl'] = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
+    parsed_config[:redis]['connection_pool_size'] = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10
+    parsed_config[:redis]['connection_pool_timeout'] = string_to_int(ENV['REDIS_CONNECTION_POOL_TIMEOUT']) || parsed_config[:redis]['connection_pool_timeout'] || 5
 
     parsed_config[:statsd] = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
     parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
@@ -117,6 +125,7 @@ module Vmpooler
 
     parsed_config[:pools].each do |pool|
       parsed_config[:pool_names] << pool['name']
+      pool['ready_ttl'] ||= parsed_config[:config]['ready_ttl']
       if pool['alias']
         if pool['alias'].is_a?(Array)
           pool['alias'].each do |pool_alias|
@@ -154,6 +163,19 @@ module Vmpooler
     pools
   end
 
+  def self.redis_connection_pool(host, port, password, size, timeout, metrics)
+    Vmpooler::PoolManager::GenericConnectionPool.new(
+      metrics: metrics,
+      metric_prefix: 'redis_connection_pool',
+      size: size,
+      timeout: timeout
+    ) do
+      connection = Concurrent::Hash.new
+      redis = new_redis(host, port, password)
+      connection['connection'] = redis
+    end
+  end
+
   def self.new_redis(host = 'localhost', port = nil, password = nil)
     Redis.new(host: host, port: port, password: password)
   end

data/lib/vmpooler/api/helpers.rb

@@ -84,29 +84,29 @@ module Vmpooler
         when 'ldap'
           ldap_base = auth[:ldap]['base']
           ldap_port = auth[:ldap]['port'] || 389
+          ldap_user_obj = auth[:ldap]['user_object']
+          ldap_host = auth[:ldap]['host']
 
-          if ldap_base.is_a? Array
-            ldap_base.each do |search_base|
+          unless ldap_base.is_a? Array
+            ldap_base = ldap_base.split
+          end
+
+          unless ldap_user_obj.is_a? Array
+            ldap_user_obj = ldap_user_obj.split
+          end
+
+          ldap_base.each do |search_base|
+            ldap_user_obj.each do |search_user_obj|
               result = authenticate_ldap(
                 ldap_port,
-                auth[:ldap]['host'],
-                auth[:ldap]['user_object'],
+                ldap_host,
+                search_user_obj,
                 search_base,
                 username_str,
                 password_str
               )
-              return true if result == true
+              return true if result
             end
-          else
-            result = authenticate_ldap(
-              ldap_port,
-              auth[:ldap]['host'],
-              auth[:ldap]['user_object'],
-              ldap_base,
-              username_str,
-              password_str
-            )
-            return result
           end
 
           return false
@@ -238,7 +238,7 @@ module Vmpooler
         queue[:running]   = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
         queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
 
-        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
+        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i
         queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
         queue[:booting] = 0 if queue[:booting] < 0
         queue[:total]   = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i

data/lib/vmpooler/api/v1.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'vmpooler/util/parsing'
+
 module Vmpooler
   class API
   class V1 < Sinatra::Base
@@ -42,6 +44,68 @@ module Vmpooler
       Vmpooler::API.settings.checkoutlock
     end
 
+    def get_template_aliases(template)
+      result = []
+      aliases = Vmpooler::API.settings.config[:alias]
+      if aliases
+        result += aliases[template] if aliases[template].is_a?(Array)
+        template_backends << aliases[template] if aliases[template].is_a?(String)
+      end
+      result
+    end
+
+    def get_pool_weights(template_backends)
+      pool_index = pool_index(pools)
+      weighted_pools = {}
+      template_backends.each do |t|
+        next unless pool_index.key? t
+
+        index = pool_index[t]
+        clone_target = pools[index]['clone_target'] || config['clone_target']
+        next unless config.key?('backend_weight')
+
+        weight = config['backend_weight'][clone_target]
+        if weight
+          weighted_pools[t] = weight
+        end
+      end
+      weighted_pools
+    end
+
+    def count_selection(selection)
+      result = {}
+      selection.uniq.each do |poolname|
+        result[poolname] = selection.count(poolname)
+      end
+      result
+    end
+
+    def evaluate_template_aliases(template, count)
+      template_backends = []
+      template_backends << template if backend.sismember('vmpooler__pools', template)
+      selection = []
+      aliases = get_template_aliases(template)
+      if aliases
+        template_backends += aliases
+        weighted_pools = get_pool_weights(template_backends)
+
+        pickup = Pickup.new(weighted_pools) if weighted_pools.count == template_backends.count
+        count.to_i.times do
+          if pickup
+            selection << pickup.pick
+          else
+            selection << template_backends.sample
+          end
+        end
+      else
+        count.to_i.times do
+          selection << template
+        end
+      end
+
+      count_selection(selection)
+    end
+
     def fetch_single_vm(template)
       template_backends = [template]
       aliases = Vmpooler::API.settings.config[:alias]
@@ -245,11 +309,9 @@ module Vmpooler
       pool_index = pool_index(pools)
       template_configs = backend.hgetall('vmpooler__config__template')
       template_configs&.each do |poolname, template|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['template'] == template
-              pools[pool_index[poolname]]['template'] = template
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['template'] = template
       end
     end
 
@@ -257,11 +319,9 @@ module Vmpooler
       pool_index = pool_index(pools)
       poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
       poolsize_configs&.each do |poolname, size|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['size'] == size.to_i
-              pools[pool_index[poolname]]['size'] == size.to_i
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['size'] = size.to_i
       end
     end
 
@@ -269,12 +329,70 @@ module Vmpooler
       pool_index = pool_index(pools)
       clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
       clone_target_configs&.each do |poolname, clone_target|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['clone_target'] == clone_target
-              pools[pool_index[poolname]]['clone_target'] == clone_target
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['clone_target'] = clone_target
+      end
+    end
+
+    def too_many_requested?(payload)
+      payload&.each do |poolname, count|
+        next unless count.to_i > config['max_ondemand_instances_per_request']
+
+        metrics.increment('ondemandrequest.toomanyrequests.' + poolname)
+        return true
+      end
+      false
+    end
+
+    def generate_ondemand_request(payload)
+      result = { 'ok': false }
+
+      requested_instances = payload.reject { |k, _v| k == 'request_id' }
+      if too_many_requested?(requested_instances)
+        result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
+        status 403
+        return result
+      end
+
+      score = Time.now.to_i
+      request_id = payload['request_id']
+      request_id ||= generate_request_id
+      result['request_id'] = request_id
+
+      if backend.exists("vmpooler__odrequest__#{request_id}")
+        result['message'] = "request_id '#{request_id}' has already been created"
+        status 409
+        metrics.increment('ondemandrequest.generate.duplicaterequests')
+        return result
+      end
+
+      status 201
+
+      platforms_with_aliases = []
+      requested_instances.each do |poolname, count|
+        selection = evaluate_template_aliases(poolname, count)
+        selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
+      end
+      platforms_string = platforms_with_aliases.join(',')
+
+      return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
+
+      backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+      if Vmpooler::API.settings.config[:auth] and has_token?
+        backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
+        backend.hset("vmpooler__odrequest__#{request_id}", 'token:user',
+                     backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user'))
       end
+
+      result['domain'] = config['domain'] if config['domain']
+      result[:ok] = true
+      metrics.increment('ondemandrequest.generate.success')
+      result
+    end
+
+    def generate_request_id
+      SecureRandom.uuid
     end
 
     get '/' do
@@ -395,7 +513,7 @@ module Vmpooler
           end
 
           # for backwards compatibility, include separate "empty" stats in "status" block
-          if ready == 0
+          if ready == 0 && max != 0
             result[:status][:empty] ||= []
             result[:status][:empty].push(pool['name'])
 
@@ -689,6 +807,88 @@ module Vmpooler
       JSON.pretty_generate(result)
     end
 
+    post "#{api_prefix}/ondemandvm/?" do
+      content_type :json
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      result = { 'ok' => false }
+
+      begin
+        payload = JSON.parse(request.body.read)
+
+        if payload
+          invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+          if invalid.empty?
+            result = generate_ondemand_request(payload)
+          else
+            result[:bad_templates] = invalid
+            invalid.each do |bad_template|
+              metrics.increment('ondemandrequest.invalid.' + bad_template)
+            end
+            status 404
+          end
+        else
+          metrics.increment('ondemandrequest.invalid.unknown')
+          status 404
+        end
+      rescue JSON::ParserError
+        status 400
+        result = {
+          'ok' => false,
+          'message' => 'JSON payload could not be parsed'
+        }
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    post "#{api_prefix}/ondemandvm/:template/?" do
+      content_type :json
+      result = { 'ok' => false }
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      payload = extract_templates_from_query_params(params[:template])
+
+      if payload
+        invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+        if invalid.empty?
+          result = generate_ondemand_request(payload)
+        else
+          result[:bad_templates] = invalid
+          invalid.each do |bad_template|
+            metrics.increment('ondemandrequest.invalid.' + bad_template)
+          end
+          status 404
+        end
+      else
+        metrics.increment('ondemandrequest.invalid.unknown')
+        status 404
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    get "#{api_prefix}/ondemandvm/:requestid/?" do
+      content_type :json
+
+      status 404
+      result = check_ondemand_request(params[:requestid])
+
+      JSON.pretty_generate(result)
+    end
+
+    delete "#{api_prefix}/ondemandvm/:requestid/?" do
+      content_type :json
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      status 404
+      result = delete_ondemand_request(params[:requestid])
+
+      JSON.pretty_generate(result)
+    end
+
     post "#{api_prefix}/vm/?" do
       content_type :json
       result = { 'ok' => false }
@@ -764,6 +964,73 @@ module Vmpooler
       invalid
     end
 
+    def check_ondemand_request(request_id)
+      result = { 'ok' => false }
+      request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
+      if request_hash.empty?
+        result['message'] = "no request found for request_id '#{request_id}'"
+        return result
+      end
+
+      result['request_id'] = request_id
+      result['ready'] = false
+      result['ok'] = true
+      status 202
+
+      if request_hash['status'] == 'ready'
+        result['ready'] = true
+        Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
+          instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+          result[platform_alias] = { 'hostname': instances }
+        end
+        result['domain'] = config['domain'] if config['domain']
+        status 200
+      elsif request_hash['status'] == 'failed'
+        result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
+        status 200
+      elsif request_hash['status'] == 'deleted'
+        result['message'] = 'The request has been deleted'
+        status 200
+      else
+        Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
+          instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+          result[platform_alias] = {
+            'ready': instance_count.to_s,
+            'pending': (count.to_i - instance_count.to_i).to_s
+          }
+        end
+      end
+
+      result
+    end
+
+    def delete_ondemand_request(request_id)
+      result = { 'ok' => false }
+
+      platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
+      unless platforms
+        result['message'] = "no request found for request_id '#{request_id}'"
+        return result
+      end
+
+      if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
+        result['message'] = 'the request has already been deleted'
+      else
+        backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
+
+        Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
+          backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
+            backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
+          end
+          backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+        end
+        backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
+      end
+      status 200
+      result['ok'] = true
+      result
+    end
+
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
@@ -923,6 +1190,7 @@ module Vmpooler
               unless arg.to_i > 0
                 failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.")
               end
+
             when 'tags'
               unless arg.is_a?(Hash)
                 failure.push("You provided tags (#{arg}) as something other than a hash.")
@@ -1047,7 +1315,7 @@ module Vmpooler
             invalid.each do |bad_template|
               metrics.increment("config.invalid.#{bad_template}")
             end
-            result[:bad_templates] = invalid
+            result[:not_configured] = invalid
             status 400
           end
         else