vmpooler 0.12.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19dd8ce83614c49e92f13e043c2ee7865f1c664d92aa8f13e47fc328eb033690
-  data.tar.gz: c3618c9c13591b1b06748ac3e5d1c2dc7ba2eb9793ea2f3f014a762fd6deb4ed
+  metadata.gz: 74df51cf1d525f0bc7afe2da930e68d0b2e1e014e10d184213556e4a13bd170b
+  data.tar.gz: 7cfe6d26ebb99e421f2b9daeabe1dfcb610cc8457cae1f79c5632df33eb6d426
 SHA512:
-  metadata.gz: 8dcb8f8765f9c1fac52c2d0088aaa09a589b5145d5a81e901af614b8fcbbd588669b3f14e5c42f4d56d4681165f8814ef11b22073df6f97b85cef6fcb10d04e2
-  data.tar.gz: d7c917233daee973486a6a8add55222a92bb1708521caaefdf2666824c0785cebd1ce7e298448a66736bf7e0646f02a560067e54cc1d06a07ca6abae645898c9
+  metadata.gz: d4e00e1cf43c08652b6186a741f048a487c1300744a2a40912759bdf06a2ad306fd59ae9adb10364be191047edc4cc84d8ea99fbf211806d59228b0993e61d0f
+  data.tar.gz: 4948f3405e5969bcd55b350604d1a2b8bfa9270960b635c4448a9478764f79bef1b2af101e11a3389587cdc2bd0fb22f28c1b7d1582227414aa42764d8451a74

data/bin/vmpooler

@@ -7,36 +7,43 @@ config = Vmpooler.config
 redis_host = config[:redis]['server']
 redis_port = config[:redis]['port']
 redis_password = config[:redis]['password']
+redis_connection_pool_size = config[:redis]['connection_pool_size']
+redis_connection_pool_timeout = config[:redis]['connection_pool_timeout']
 logger_file = config[:config]['logfile']
 
-metrics = Vmpooler.new_metrics(config)
+logger = Vmpooler::Logger.new logger_file
+metrics = Vmpooler::Metrics.init(logger, config)
 
 torun_threads = []
 if ARGV.count == 0
-  torun = ['api', 'manager']
+  torun = %i[api manager]
 else
   torun = []
-  torun << 'api' if ARGV.include? 'api'
-  torun << 'manager' if ARGV.include? 'manager'
+  torun << :api if ARGV.include? 'api'
+  torun << :manager if ARGV.include? 'manager'
   exit(2) if torun.empty?
 end
 
-if torun.include? 'api'
+if torun.include? :api
   api = Thread.new do
-    thr = Vmpooler::API.new
     redis = Vmpooler.new_redis(redis_host, redis_port, redis_password)
-    thr.helpers.configure(config, redis, metrics)
-    thr.helpers.execute!
+    Vmpooler::API.execute(torun, config, redis, metrics, logger)
   end
   torun_threads << api
+elsif metrics.respond_to?(:setup_prometheus_metrics)
+  # Run the cut down API - Prometheus Metrics only.
+  prometheus_only_api = Thread.new do
+    Vmpooler::API.execute(torun, config, nil, metrics, logger)
+  end
+  torun_threads << prometheus_only_api
 end
 
-if torun.include? 'manager'
+if torun.include? :manager
   manager = Thread.new do
     Vmpooler::PoolManager.new(
       config,
-      Vmpooler.new_logger(logger_file),
-      Vmpooler.new_redis(redis_host, redis_port, redis_password),
+      logger,
+      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics),
       metrics
     ).execute!
   end

data/lib/vmpooler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Vmpooler
+  require 'concurrent'
   require 'date'
   require 'json'
   require 'net/ldap'
@@ -14,7 +15,7 @@ module Vmpooler
   require 'timeout'
   require 'yaml'
 
-  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
+  %w[api metrics logger pool_manager generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"
   end
 
@@ -58,9 +59,14 @@ module Vmpooler
 
     # Set some configuration defaults
     parsed_config[:config]['task_limit'] = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
+    parsed_config[:config]['ondemand_clone_limit'] = string_to_int(ENV['ONDEMAND_CLONE_LIMIT']) || parsed_config[:config]['ondemand_clone_limit'] || 10
+    parsed_config[:config]['max_ondemand_instances_per_request'] = string_to_int(ENV['MAX_ONDEMAND_INSTANCES_PER_REQUEST']) || parsed_config[:config]['max_ondemand_instances_per_request'] || 10
     parsed_config[:config]['migration_limit'] = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
     parsed_config[:config]['vm_checktime'] = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
     parsed_config[:config]['vm_lifetime'] = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24
+    parsed_config[:config]['max_lifetime_upper_limit'] = string_to_int(ENV['MAX_LIFETIME_UPPER_LIMIT']) || parsed_config[:config]['max_lifetime_upper_limit']
+    parsed_config[:config]['ready_ttl'] = string_to_int(ENV['READY_TTL']) || parsed_config[:config]['ready_ttl'] || 60
+    parsed_config[:config]['ondemand_request_ttl'] = string_to_int(ENV['ONDEMAND_REQUEST_TTL']) || parsed_config[:config]['ondemand_request_ttl'] || 5
     parsed_config[:config]['prefix'] = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
 
     parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE']
@@ -72,18 +78,21 @@ module Vmpooler
     parsed_config[:config]['vm_lifetime_auth'] = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
     parsed_config[:config]['max_tries'] = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
     parsed_config[:config]['retry_factor'] = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
-    parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
+    parsed_config[:config]['create_folders'] = true?(ENV['CREATE_FOLDERS']) if ENV['CREATE_FOLDERS']
     parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
     set_linked_clone(parsed_config)
     parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
     parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
     parsed_config[:config]['usage_stats'] = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
+    parsed_config[:config]['request_logger'] = ENV['REQUEST_LOGGER'] if ENV['REQUEST_LOGGER']
 
     parsed_config[:redis] = parsed_config[:redis] || {}
     parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
     parsed_config[:redis]['port'] = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT']
     parsed_config[:redis]['password'] = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
     parsed_config[:redis]['data_ttl'] = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
+    parsed_config[:redis]['connection_pool_size'] = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10
+    parsed_config[:redis]['connection_pool_timeout'] = string_to_int(ENV['REDIS_CONNECTION_POOL_TIMEOUT']) || parsed_config[:redis]['connection_pool_timeout'] || 5
 
     parsed_config[:statsd] = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
     parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
@@ -117,6 +126,7 @@ module Vmpooler
 
     parsed_config[:pools].each do |pool|
       parsed_config[:pool_names] << pool['name']
+      pool['ready_ttl'] ||= parsed_config[:config]['ready_ttl']
       if pool['alias']
         if pool['alias'].is_a?(Array)
           pool['alias'].each do |pool_alias|
@@ -154,22 +164,22 @@ module Vmpooler
     pools
   end
 
-  def self.new_redis(host = 'localhost', port = nil, password = nil)
-    Redis.new(host: host, port: port, password: password)
-  end
-
-  def self.new_logger(logfile)
-    Vmpooler::Logger.new logfile
+  def self.redis_connection_pool(host, port, password, size, timeout, metrics)
+    Vmpooler::PoolManager::GenericConnectionPool.new(
+      metrics: metrics,
+      connpool_type: 'redis_connection_pool',
+      connpool_provider: 'manager',
+      size: size,
+      timeout: timeout
+    ) do
+      connection = Concurrent::Hash.new
+      redis = new_redis(host, port, password)
+      connection['connection'] = redis
+    end
   end
 
-  def self.new_metrics(params)
-    if params[:statsd]
-      Vmpooler::Statsd.new(params[:statsd])
-    elsif params[:graphite]
-      Vmpooler::Graphite.new(params[:graphite])
-    else
-      Vmpooler::DummyStatsd.new
-    end
+  def self.new_redis(host = 'localhost', port = nil, password = nil)
+    Redis.new(host: host, port: port, password: password)
   end
 
   def self.pools(conf)

data/lib/vmpooler/api.rb

@@ -2,51 +2,58 @@
 
 module Vmpooler
   class API < Sinatra::Base
-    def initialize
-      super
+    # Load API components
+    %w[helpers dashboard reroute v1 request_logger].each do |lib|
+      require "vmpooler/api/#{lib}"
     end
+    # Load dashboard components
+    require 'vmpooler/dashboard'
 
-    not_found do
-      content_type :json
+    def self.execute(torun, config, redis, metrics, logger)
+      self.settings.set :config, config
+      self.settings.set :redis, redis unless redis.nil?
+      self.settings.set :metrics, metrics
+      self.settings.set :checkoutlock, Mutex.new
 
-      result = {
-        ok: false
-      }
+      # Deflating in all situations
+      # https://www.schneems.com/2017/11/08/80-smaller-rails-footprint-with-rack-deflate/
+      use Rack::Deflater
 
-      JSON.pretty_generate(result)
-    end
+      # not_found clause placed here to fix rspec test issue.
+      not_found do
+        content_type :json
 
-    # Load dashboard components
-    begin
-      require 'dashboard'
-    rescue LoadError
-      require File.expand_path(File.join(File.dirname(__FILE__), 'dashboard'))
-    end
+        result = {
+          ok: false
+        }
 
-    use Vmpooler::Dashboard
+        JSON.pretty_generate(result)
+      end
 
-    # Load API components
-    %w[helpers dashboard reroute v1].each do |lib|
-      begin
-        require "api/#{lib}"
-      rescue LoadError
-        require File.expand_path(File.join(File.dirname(__FILE__), 'api', lib))
+      if metrics.respond_to?(:setup_prometheus_metrics)
+        # Prometheus metrics are only setup if actually specified
+        # in the config file.
+        metrics.setup_prometheus_metrics(torun)
+
+        # Using customised collector that filters out hostnames on API paths
+        require 'vmpooler/metrics/promstats/collector_middleware'
+        require 'prometheus/middleware/exporter'
+        use Vmpooler::Metrics::Promstats::CollectorMiddleware, metrics_prefix: "#{metrics.metrics_prefix}_http"
+        use Prometheus::Middleware::Exporter, path: metrics.endpoint
       end
-    end
 
-    use Vmpooler::API::Dashboard
-    use Vmpooler::API::Reroute
-    use Vmpooler::API::V1
+      if torun.include? :api
+        # Enable API request logging only if required
+        use Vmpooler::API::RequestLogger, logger: logger if config[:config]['request_logger']
 
-    def configure(config, redis, metrics)
-      self.settings.set :config, config
-      self.settings.set :redis, redis
-      self.settings.set :metrics, metrics
-      self.settings.set :checkoutlock, Mutex.new
-    end
+        use Vmpooler::Dashboard
+        use Vmpooler::API::Dashboard
+        use Vmpooler::API::Reroute
+        use Vmpooler::API::V1
+      end
 
-    def execute!
-      self.settings.run!
+      # Get thee started O WebServer
+      self.run!
     end
   end
 end

data/lib/vmpooler/api/helpers.rb

@@ -13,7 +13,7 @@ module Vmpooler
       def valid_token?(backend)
         return false unless has_token?
 
-        backend.exists('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN']) ? true : false
+        backend.exists?('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN']) ? true : false
       end
 
       def validate_token(backend)
@@ -238,7 +238,7 @@ module Vmpooler
         queue[:running]   = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
         queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
 
-        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
+        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i + backend.get('vmpooler__tasks__ondemandclone').to_i
         queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
         queue[:booting] = 0 if queue[:booting] < 0
         queue[:total]   = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i

data/lib/vmpooler/api/request_logger.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  class API
+    class RequestLogger
+      attr_reader :app
+
+      def initialize(app, options = {})
+        @app = app
+        @logger = options[:logger]
+      end
+
+      def call(env)
+        status, headers, body = @app.call(env)
+        @logger.log('s', "[ ] API: Method: #{env['REQUEST_METHOD']}, Status: #{status}, Path: #{env['PATH_INFO']}, Body: #{body}")
+        [status, headers, body]
+      end
+    end
+  end
+end

data/lib/vmpooler/api/v1.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'vmpooler/util/parsing'
+
 module Vmpooler
   class API
   class V1 < Sinatra::Base
@@ -42,6 +44,68 @@ module Vmpooler
       Vmpooler::API.settings.checkoutlock
     end
 
+    def get_template_aliases(template)
+      result = []
+      aliases = Vmpooler::API.settings.config[:alias]
+      if aliases
+        result += aliases[template] if aliases[template].is_a?(Array)
+        template_backends << aliases[template] if aliases[template].is_a?(String)
+      end
+      result
+    end
+
+    def get_pool_weights(template_backends)
+      pool_index = pool_index(pools)
+      weighted_pools = {}
+      template_backends.each do |t|
+        next unless pool_index.key? t
+
+        index = pool_index[t]
+        clone_target = pools[index]['clone_target'] || config['clone_target']
+        next unless config.key?('backend_weight')
+
+        weight = config['backend_weight'][clone_target]
+        if weight
+          weighted_pools[t] = weight
+        end
+      end
+      weighted_pools
+    end
+
+    def count_selection(selection)
+      result = {}
+      selection.uniq.each do |poolname|
+        result[poolname] = selection.count(poolname)
+      end
+      result
+    end
+
+    def evaluate_template_aliases(template, count)
+      template_backends = []
+      template_backends << template if backend.sismember('vmpooler__pools', template)
+      selection = []
+      aliases = get_template_aliases(template)
+      if aliases
+        template_backends += aliases
+        weighted_pools = get_pool_weights(template_backends)
+
+        pickup = Pickup.new(weighted_pools) if weighted_pools.count == template_backends.count
+        count.to_i.times do
+          if pickup
+            selection << pickup.pick
+          else
+            selection << template_backends.sample
+          end
+        end
+      else
+        count.to_i.times do
+          selection << template
+        end
+      end
+
+      count_selection(selection)
+    end
+
     def fetch_single_vm(template)
       template_backends = [template]
       aliases = Vmpooler::API.settings.config[:alias]
@@ -245,11 +309,9 @@ module Vmpooler
       pool_index = pool_index(pools)
       template_configs = backend.hgetall('vmpooler__config__template')
       template_configs&.each do |poolname, template|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['template'] == template
-              pools[pool_index[poolname]]['template'] = template
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['template'] = template
       end
     end
 
@@ -257,11 +319,9 @@ module Vmpooler
       pool_index = pool_index(pools)
       poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
       poolsize_configs&.each do |poolname, size|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['size'] == size.to_i
-              pools[pool_index[poolname]]['size'] == size.to_i
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['size'] = size.to_i
       end
     end
 
@@ -269,12 +329,70 @@ module Vmpooler
       pool_index = pool_index(pools)
       clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
       clone_target_configs&.each do |poolname, clone_target|
-          if pool_index.include? poolname
-            unless pools[pool_index[poolname]]['clone_target'] == clone_target
-              pools[pool_index[poolname]]['clone_target'] == clone_target
-            end
-          end
+        next unless pool_index.include? poolname
+
+        pools[pool_index[poolname]]['clone_target'] = clone_target
+      end
+    end
+
+    def too_many_requested?(payload)
+      payload&.each do |poolname, count|
+        next unless count.to_i > config['max_ondemand_instances_per_request']
+
+        metrics.increment('ondemandrequest_fail.toomanyrequests.' + poolname)
+        return true
+      end
+      false
+    end
+
+    def generate_ondemand_request(payload)
+      result = { 'ok': false }
+
+      requested_instances = payload.reject { |k, _v| k == 'request_id' }
+      if too_many_requested?(requested_instances)
+        result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
+        status 403
+        return result
+      end
+
+      score = Time.now.to_i
+      request_id = payload['request_id']
+      request_id ||= generate_request_id
+      result['request_id'] = request_id
+
+      if backend.exists?("vmpooler__odrequest__#{request_id}")
+        result['message'] = "request_id '#{request_id}' has already been created"
+        status 409
+        metrics.increment('ondemandrequest_generate.duplicaterequests')
+        return result
+      end
+
+      status 201
+
+      platforms_with_aliases = []
+      requested_instances.each do |poolname, count|
+        selection = evaluate_template_aliases(poolname, count)
+        selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
+      end
+      platforms_string = platforms_with_aliases.join(',')
+
+      return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
+
+      backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
+      if Vmpooler::API.settings.config[:auth] and has_token?
+        backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
+        backend.hset("vmpooler__odrequest__#{request_id}", 'token:user',
+                     backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user'))
       end
+
+      result['domain'] = config['domain'] if config['domain']
+      result[:ok] = true
+      metrics.increment('ondemandrequest_generate.success')
+      result
+    end
+
+    def generate_request_id
+      SecureRandom.uuid
     end
 
     get '/' do
@@ -395,7 +513,7 @@ module Vmpooler
           end
 
           # for backwards compatibility, include separate "empty" stats in "status" block
-          if ready == 0
+          if ready == 0 && max != 0
             result[:status][:empty] ||= []
             result[:status][:empty].push(pool['name'])
 
@@ -689,9 +807,96 @@ module Vmpooler
       JSON.pretty_generate(result)
     end
 
+    post "#{api_prefix}/ondemandvm/?" do
+      content_type :json
+      metrics.increment('api_vm.post.ondemand.requestid')
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      result = { 'ok' => false }
+
+      begin
+        payload = JSON.parse(request.body.read)
+
+        if payload
+          invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+          if invalid.empty?
+            result = generate_ondemand_request(payload)
+          else
+            result[:bad_templates] = invalid
+            invalid.each do |bad_template|
+              metrics.increment('ondemandrequest_fail.invalid.' + bad_template)
+            end
+            status 404
+          end
+        else
+          metrics.increment('ondemandrequest_fail.invalid.unknown')
+          status 404
+        end
+      rescue JSON::ParserError
+        status 400
+        result = {
+          'ok' => false,
+          'message' => 'JSON payload could not be parsed'
+        }
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    post "#{api_prefix}/ondemandvm/:template/?" do
+      content_type :json
+      result = { 'ok' => false }
+      metrics.increment('api_vm.delete.ondemand.template')
+
+      need_token! if Vmpooler::API.settings.config[:auth]
+
+      payload = extract_templates_from_query_params(params[:template])
+
+      if payload
+        invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
+        if invalid.empty?
+          result = generate_ondemand_request(payload)
+        else
+          result[:bad_templates] = invalid
+          invalid.each do |bad_template|
+            metrics.increment('ondemandrequest_fail.invalid.' + bad_template)
+          end
+          status 404
+        end
+      else
+        metrics.increment('ondemandrequest_fail.invalid.unknown')
+        status 404
+      end
+
+      JSON.pretty_generate(result)
+    end
+
+    get "#{api_prefix}/ondemandvm/:requestid/?" do
+      content_type :json
+      metrics.increment('api_vm.get.ondemand.request')
+
+      status 404
+      result = check_ondemand_request(params[:requestid])
+
+      JSON.pretty_generate(result)
+    end
+
+    delete "#{api_prefix}/ondemandvm/:requestid/?" do
+      content_type :json
+      need_token! if Vmpooler::API.settings.config[:auth]
+      metrics.increment('api_vm.delete.ondemand.request')
+
+      status 404
+      result = delete_ondemand_request(params[:requestid])
+
+      JSON.pretty_generate(result)
+    end
+
     post "#{api_prefix}/vm/?" do
       content_type :json
       result = { 'ok' => false }
+      metrics.increment('api_vm.post.vm.checkout')
 
       payload = JSON.parse(request.body.read)
 
@@ -764,9 +969,77 @@ module Vmpooler
       invalid
     end
 
+    def check_ondemand_request(request_id)
+      result = { 'ok' => false }
+      request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
+      if request_hash.empty?
+        result['message'] = "no request found for request_id '#{request_id}'"
+        return result
+      end
+
+      result['request_id'] = request_id
+      result['ready'] = false
+      result['ok'] = true
+      status 202
+
+      if request_hash['status'] == 'ready'
+        result['ready'] = true
+        Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
+          instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+          result[platform_alias] = { 'hostname': instances }
+        end
+        result['domain'] = config['domain'] if config['domain']
+        status 200
+      elsif request_hash['status'] == 'failed'
+        result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
+        status 200
+      elsif request_hash['status'] == 'deleted'
+        result['message'] = 'The request has been deleted'
+        status 200
+      else
+        Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
+          instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+          result[platform_alias] = {
+            'ready': instance_count.to_s,
+            'pending': (count.to_i - instance_count.to_i).to_s
+          }
+        end
+      end
+
+      result
+    end
+
+    def delete_ondemand_request(request_id)
+      result = { 'ok' => false }
+
+      platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
+      unless platforms
+        result['message'] = "no request found for request_id '#{request_id}'"
+        return result
+      end
+
+      if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
+        result['message'] = 'the request has already been deleted'
+      else
+        backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
+
+        Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
+          backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
+            backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
+          end
+          backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
+        end
+        backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
+      end
+      status 200
+      result['ok'] = true
+      result
+    end
+
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
+      metrics.increment('api_vm.get.vm.template')
 
       payload = extract_templates_from_query_params(params[:template])
 
@@ -790,6 +1063,7 @@ module Vmpooler
 
     get "#{api_prefix}/vm/:hostname/?" do
       content_type :json
+      metrics.increment('api_vm.get.vm.hostname')
 
       result = {}
 
@@ -862,6 +1136,7 @@ module Vmpooler
 
     delete "#{api_prefix}/vm/:hostname/?" do
       content_type :json
+      metrics.increment('api_vm.delete.vm.hostname')
 
       result = {}
 
@@ -879,8 +1154,9 @@ module Vmpooler
 
           status 200
           result['ok'] = true
+          metrics.increment('delete.success')
         else
-          metrics.increment('delete.srem.failed')
+          metrics.increment('delete.failed')
         end
       end
 
@@ -889,6 +1165,7 @@ module Vmpooler
 
     put "#{api_prefix}/vm/:hostname/?" do
       content_type :json
+      metrics.increment('api_vm.put.vm.modify')
 
       status 404
       result = { 'ok' => false }
@@ -897,7 +1174,7 @@ module Vmpooler
 
       params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
 
-      if backend.exists('vmpooler__vm__' + params[:hostname])
+      if backend.exists?('vmpooler__vm__' + params[:hostname])
         begin
           jdata = JSON.parse(request.body.read)
         rescue StandardError
@@ -923,6 +1200,7 @@ module Vmpooler
               unless arg.to_i > 0
                 failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.")
               end
+
             when 'tags'
               unless arg.is_a?(Hash)
                 failure.push("You provided tags (#{arg}) as something other than a hash.")
@@ -964,6 +1242,7 @@ module Vmpooler
 
     post "#{api_prefix}/vm/:hostname/disk/:size/?" do
       content_type :json
+      metrics.increment('api_vm.post.vm.disksize')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 
@@ -972,7 +1251,7 @@ module Vmpooler
 
       params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
 
-      if ((params[:size].to_i > 0 )and (backend.exists('vmpooler__vm__' + params[:hostname])))
+      if ((params[:size].to_i > 0 )and (backend.exists?('vmpooler__vm__' + params[:hostname])))
         result[params[:hostname]] = {}
         result[params[:hostname]]['disk'] = "+#{params[:size]}gb"
 
@@ -987,6 +1266,7 @@ module Vmpooler
 
     post "#{api_prefix}/vm/:hostname/snapshot/?" do
       content_type :json
+      metrics.increment('api_vm.post.vm.snapshot')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 
@@ -995,7 +1275,7 @@ module Vmpooler
 
       params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
 
-      if backend.exists('vmpooler__vm__' + params[:hostname])
+      if backend.exists?('vmpooler__vm__' + params[:hostname])
         result[params[:hostname]] = {}
 
         o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
@@ -1012,6 +1292,7 @@ module Vmpooler
 
     post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
       content_type :json
+      metrics.increment('api_vm.post.vm.disksize')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 
@@ -1047,7 +1328,7 @@ module Vmpooler
             invalid.each do |bad_template|
               metrics.increment("config.invalid.#{bad_template}")
             end
-            result[:bad_templates] = invalid
+            result[:not_configured] = invalid
             status 400
           end
         else