vmpooler 0.14.2 → 0.14.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b0e071edb016f25c0379ebc2d7768e945096333d9be2f1f31ce66fc32474fd1
-  data.tar.gz: 1bbac7250090c48a74bf671d316cb9aa45d1dc55af1f69351e9afd27f7c4ed9d
+  metadata.gz: aa377464c0b9bfa6ac09d96064dceb410417ddcfc81e2523e5d32e691f7e8091
+  data.tar.gz: d61f1c0ea6fe636b593912cbe26954edd253b07952b999205adf6b9271c8ae3b
 SHA512:
-  metadata.gz: 21bc1b2a3c83e82a2e4f3ad6f54a41a380a76a331ca8613b49acf0f7cd2df6ff69162e600f7d449e482e110d1ac7e3e5d9534cd1479d990dd4ff4f8c2c079b74
-  data.tar.gz: 109ae1cbfa75cc80a3afff19938677856f9ac0fd858cb546d6ea42b8dbbddf515cad3cdb6cfcb340addbe863c5ee1fc9c4025704bdb298584a7ab8696dd27de9
+  metadata.gz: 1f5c68048317e9374725e4c5cf71c53d32c28ad3e287b7f82c54a2bb852e039f21b8326fa62160f77dcf7572ed39b4035a005bf214097634844052bab59c828a
+  data.tar.gz: 877db96232be8617987096b227f390a643b788d50971ebc629de7f5aca720cd96202d8d0292ecffa8a7fe2e366ee649f9e2eda59202423929abd94f90beae1cd

data/bin/vmpooler

@@ -2,29 +2,39 @@
 # frozen_string_literal: true
 
 require 'vmpooler'
+require 'vmpooler/version'
 
 config = Vmpooler.config
+logger_file = config[:config]['logfile']
+prefix = config[:config]['prefix']
 redis_host = config[:redis]['server']
 redis_port = config[:redis]['port']
 redis_password = config[:redis]['password']
 redis_connection_pool_size = config[:redis]['connection_pool_size']
 redis_connection_pool_timeout = config[:redis]['connection_pool_timeout']
-logger_file = config[:config]['logfile']
+redis_reconnect_attempts = config[:redis]['reconnect_attempts']
+tracing_enabled = config[:tracing]['enabled']
+tracing_jaeger_host = config[:tracing]['jaeger_host']
 
 logger = Vmpooler::Logger.new logger_file
 metrics = Vmpooler::Metrics.init(logger, config)
 
+version = Vmpooler::VERSION
+
+startup_args = ARGV
+Vmpooler.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version)
+
 torun_threads = []
 if ARGV.count == 0
   torun = %i[api manager]
 else
   torun = []
-  torun << :api if ARGV.include? 'api'
-  torun << :manager if ARGV.include? 'manager'
+  torun << :api if ARGV.include?('api')
+  torun << :manager if ARGV.include?('manager')
   exit(2) if torun.empty?
 end
 
-if torun.include? :api
+if torun.include?(:api)
   api = Thread.new do
     redis = Vmpooler.new_redis(redis_host, redis_port, redis_password)
     Vmpooler::API.execute(torun, config, redis, metrics, logger)
@@ -38,12 +48,12 @@ elsif metrics.respond_to?(:setup_prometheus_metrics)
   torun_threads << prometheus_only_api
 end
 
-if torun.include? :manager
+if torun.include?(:manager)
   manager = Thread.new do
     Vmpooler::PoolManager.new(
       config,
       logger,
-      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics),
+      Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics, redis_reconnect_attempts),
       metrics
     ).execute!
   end

data/lib/vmpooler.rb

@@ -15,6 +15,15 @@ module Vmpooler
   require 'timeout'
   require 'yaml'
 
+  # Dependencies for tracing
+  require 'opentelemetry-api'
+  require 'opentelemetry-instrumentation-concurrent_ruby'
+  require 'opentelemetry-instrumentation-redis'
+  require 'opentelemetry-instrumentation-sinatra'
+  require 'opentelemetry-sdk'
+  require 'opentelemetry/exporter/jaeger'
+  require 'opentelemetry/resource/detectors'
+
   %w[api metrics logger pool_manager generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"
   end
@@ -58,59 +67,62 @@ module Vmpooler
     end
 
     # Set some configuration defaults
-    parsed_config[:config]['task_limit'] = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
-    parsed_config[:config]['ondemand_clone_limit'] = string_to_int(ENV['ONDEMAND_CLONE_LIMIT']) || parsed_config[:config]['ondemand_clone_limit'] || 10
+    parsed_config[:config]['task_limit']                         = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
+    parsed_config[:config]['ondemand_clone_limit']               = string_to_int(ENV['ONDEMAND_CLONE_LIMIT']) || parsed_config[:config]['ondemand_clone_limit'] || 10
     parsed_config[:config]['max_ondemand_instances_per_request'] = string_to_int(ENV['MAX_ONDEMAND_INSTANCES_PER_REQUEST']) || parsed_config[:config]['max_ondemand_instances_per_request'] || 10
-    parsed_config[:config]['migration_limit'] = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
-    parsed_config[:config]['vm_checktime'] = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
-    parsed_config[:config]['vm_lifetime'] = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24
-    parsed_config[:config]['max_lifetime_upper_limit'] = string_to_int(ENV['MAX_LIFETIME_UPPER_LIMIT']) || parsed_config[:config]['max_lifetime_upper_limit']
-    parsed_config[:config]['ready_ttl'] = string_to_int(ENV['READY_TTL']) || parsed_config[:config]['ready_ttl'] || 60
-    parsed_config[:config]['ondemand_request_ttl'] = string_to_int(ENV['ONDEMAND_REQUEST_TTL']) || parsed_config[:config]['ondemand_request_ttl'] || 5
-    parsed_config[:config]['prefix'] = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
-
-    parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE']
-
-    parsed_config[:config]['site_name'] = ENV['SITE_NAME'] if ENV['SITE_NAME']
-    parsed_config[:config]['domain'] = ENV['DOMAIN'] if ENV['DOMAIN']
-    parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
-    parsed_config[:config]['timeout'] = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
-    parsed_config[:config]['vm_lifetime_auth'] = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
-    parsed_config[:config]['max_tries'] = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
-    parsed_config[:config]['retry_factor'] = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
-    parsed_config[:config]['create_folders'] = true?(ENV['CREATE_FOLDERS']) if ENV['CREATE_FOLDERS']
-    parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
+    parsed_config[:config]['migration_limit']                    = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
+    parsed_config[:config]['vm_checktime']                       = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
+    parsed_config[:config]['vm_lifetime']                        = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24
+    parsed_config[:config]['max_lifetime_upper_limit']           = string_to_int(ENV['MAX_LIFETIME_UPPER_LIMIT']) || parsed_config[:config]['max_lifetime_upper_limit']
+    parsed_config[:config]['ready_ttl']                          = string_to_int(ENV['READY_TTL']) || parsed_config[:config]['ready_ttl'] || 60
+    parsed_config[:config]['ondemand_request_ttl']               = string_to_int(ENV['ONDEMAND_REQUEST_TTL']) || parsed_config[:config]['ondemand_request_ttl'] || 5
+    parsed_config[:config]['prefix']                             = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
+    parsed_config[:config]['logfile']                            = ENV['LOGFILE'] if ENV['LOGFILE']
+    parsed_config[:config]['site_name']                          = ENV['SITE_NAME'] if ENV['SITE_NAME']
+    parsed_config[:config]['domain']                             = ENV['DOMAIN'] if ENV['DOMAIN']
+    parsed_config[:config]['clone_target']                       = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
+    parsed_config[:config]['timeout']                            = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
+    parsed_config[:config]['vm_lifetime_auth']                   = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
+    parsed_config[:config]['max_tries']                          = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
+    parsed_config[:config]['retry_factor']                       = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
+    parsed_config[:config]['create_folders']                     = true?(ENV['CREATE_FOLDERS']) if ENV['CREATE_FOLDERS']
+    parsed_config[:config]['experimental_features']              = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
+    parsed_config[:config]['purge_unconfigured_folders']         = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
+    parsed_config[:config]['usage_stats']                        = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
+    parsed_config[:config]['request_logger']                     = ENV['REQUEST_LOGGER'] if ENV['REQUEST_LOGGER']
+    parsed_config[:config]['create_template_delta_disks']        = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
     set_linked_clone(parsed_config)
-    parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
-    parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
-    parsed_config[:config]['usage_stats'] = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
-    parsed_config[:config]['request_logger'] = ENV['REQUEST_LOGGER'] if ENV['REQUEST_LOGGER']
-
-    parsed_config[:redis] = parsed_config[:redis] || {}
-    parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
-    parsed_config[:redis]['port'] = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT']
-    parsed_config[:redis]['password'] = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
-    parsed_config[:redis]['data_ttl'] = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
-    parsed_config[:redis]['connection_pool_size'] = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10
+
+    parsed_config[:redis]                            = parsed_config[:redis] || {}
+    parsed_config[:redis]['server']                  = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
+    parsed_config[:redis]['port']                    = string_to_int(ENV['REDIS_PORT']) if ENV['REDIS_PORT']
+    parsed_config[:redis]['password']                = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
+    parsed_config[:redis]['data_ttl']                = string_to_int(ENV['REDIS_DATA_TTL']) || parsed_config[:redis]['data_ttl'] || 168
+    parsed_config[:redis]['connection_pool_size']    = string_to_int(ENV['REDIS_CONNECTION_POOL_SIZE']) || parsed_config[:redis]['connection_pool_size'] || 10
     parsed_config[:redis]['connection_pool_timeout'] = string_to_int(ENV['REDIS_CONNECTION_POOL_TIMEOUT']) || parsed_config[:redis]['connection_pool_timeout'] || 5
+    parsed_config[:redis]['reconnect_attempts']      = string_to_int(ENV['REDIS_RECONNECT_ATTEMPTS']) || parsed_config[:redis]['reconnect_attempts'] || 10
 
-    parsed_config[:statsd] = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
+    parsed_config[:statsd]           = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
     parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
     parsed_config[:statsd]['prefix'] = ENV['STATSD_PREFIX'] if ENV['STATSD_PREFIX']
-    parsed_config[:statsd]['port'] = string_to_int(ENV['STATSD_PORT']) if ENV['STATSD_PORT']
+    parsed_config[:statsd]['port']   = string_to_int(ENV['STATSD_PORT']) if ENV['STATSD_PORT']
 
-    parsed_config[:graphite] = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
+    parsed_config[:graphite]           = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
     parsed_config[:graphite]['server'] = ENV['GRAPHITE_SERVER'] if ENV['GRAPHITE_SERVER']
     parsed_config[:graphite]['prefix'] = ENV['GRAPHITE_PREFIX'] if ENV['GRAPHITE_PREFIX']
-    parsed_config[:graphite]['port'] = string_to_int(ENV['GRAPHITE_PORT']) if ENV['GRAPHITE_PORT']
+    parsed_config[:graphite]['port']   = string_to_int(ENV['GRAPHITE_PORT']) if ENV['GRAPHITE_PORT']
+
+    parsed_config[:tracing]                = parsed_config[:tracing] || {}
+    parsed_config[:tracing]['enabled']     = ENV['VMPOOLER_TRACING_ENABLED'] || parsed_config[:tracing]['enabled'] || 'false'
+    parsed_config[:tracing]['jaeger_host'] = ENV['VMPOOLER_TRACING_JAEGER_HOST'] || parsed_config[:tracing]['jaeger_host'] || 'http://localhost:14268/api/traces'
 
     parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
     if parsed_config.key? :auth
-      parsed_config[:auth]['provider'] = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
-      parsed_config[:auth][:ldap] = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
-      parsed_config[:auth][:ldap]['host'] = ENV['LDAP_HOST'] if ENV['LDAP_HOST']
-      parsed_config[:auth][:ldap]['port'] = string_to_int(ENV['LDAP_PORT']) if ENV['LDAP_PORT']
-      parsed_config[:auth][:ldap]['base'] = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
+      parsed_config[:auth]['provider']           = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
+      parsed_config[:auth][:ldap]                = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
+      parsed_config[:auth][:ldap]['host']        = ENV['LDAP_HOST'] if ENV['LDAP_HOST']
+      parsed_config[:auth][:ldap]['port']        = string_to_int(ENV['LDAP_PORT']) if ENV['LDAP_PORT']
+      parsed_config[:auth][:ldap]['base']        = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
       parsed_config[:auth][:ldap]['user_object'] = ENV['LDAP_USER_OBJECT'] if ENV['LDAP_USER_OBJECT']
     end
 
@@ -164,7 +176,7 @@ module Vmpooler
     pools
   end
 
-  def self.redis_connection_pool(host, port, password, size, timeout, metrics)
+  def self.redis_connection_pool(host, port, password, size, timeout, metrics, redis_reconnect_attempts = 0)
     Vmpooler::PoolManager::GenericConnectionPool.new(
       metrics: metrics,
       connpool_type: 'redis_connection_pool',
@@ -173,13 +185,14 @@ module Vmpooler
       timeout: timeout
     ) do
       connection = Concurrent::Hash.new
-      redis = new_redis(host, port, password)
+      redis = new_redis(host, port, password, redis_reconnect_attempts)
       connection['connection'] = redis
     end
   end
 
-  def self.new_redis(host = 'localhost', port = nil, password = nil)
-    Redis.new(host: host, port: port, password: password)
+  def self.new_redis(host = 'localhost', port = nil, password = nil, redis_reconnect_attempts = 10)
+    Redis.new(host: host, port: port, password: password, reconnect_attempts: redis_reconnect_attempts, reconnect_delay: 1.5,
+              reconnect_delay_max: 10.0)
   end
 
   def self.pools(conf)
@@ -213,4 +226,44 @@ module Vmpooler
     parsed_config[:config]['create_linked_clones'] = ENV['CREATE_LINKED_CLONES'] if ENV['CREATE_LINKED_CLONES'] =~ /true|false/
     parsed_config[:config]['create_linked_clones'] = true?(parsed_config[:config]['create_linked_clones']) if parsed_config[:config]['create_linked_clones']
   end
+
+  def self.configure_tracing(startup_args, prefix, tracing_enabled, tracing_jaeger_host, version)
+    if startup_args.length == 1 && startup_args.include?('api')
+      service_name = 'vmpooler-api'
+    elsif startup_args.length == 1 && startup_args.include?('manager')
+      service_name = 'vmpooler-manager'
+    else
+      service_name = 'vmpooler'
+    end
+
+    service_name += "-#{prefix}" unless prefix.empty?
+
+    if tracing_enabled.eql?('false')
+      puts "Exporting of traces has been disabled so the span processor has been se to a 'NoopSpanExporter'"
+      span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+        exporter: OpenTelemetry::SDK::Trace::Export::NoopSpanExporter.new
+      )
+    else
+      puts "Exporting of traces will be done over HTTP in binary Thrift format to #{tracing_jaeger_host}"
+      span_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+        exporter: OpenTelemetry::Exporter::Jaeger::CollectorExporter.new(endpoint: tracing_jaeger_host)
+      )
+    end
+
+    OpenTelemetry::SDK.configure do |c|
+      c.use 'OpenTelemetry::Instrumentation::Sinatra'
+      c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby'
+      c.use 'OpenTelemetry::Instrumentation::Redis'
+
+      c.add_span_processor(span_processor)
+
+      c.resource = OpenTelemetry::Resource::Detectors::AutoDetector.detect
+      c.resource = OpenTelemetry::SDK::Resources::Resource.create(
+        {
+          OpenTelemetry::SDK::Resources::Constants::SERVICE_RESOURCE[:name] => service_name,
+          OpenTelemetry::SDK::Resources::Constants::SERVICE_RESOURCE[:version] => version
+        }
+      )
+    end
+  end
 end

data/lib/vmpooler/api.rb

@@ -38,8 +38,8 @@ module Vmpooler
         # Using customised collector that filters out hostnames on API paths
         require 'vmpooler/metrics/promstats/collector_middleware'
         require 'prometheus/middleware/exporter'
-        use Vmpooler::Metrics::Promstats::CollectorMiddleware, metrics_prefix: "#{metrics.metrics_prefix}_http"
-        use Prometheus::Middleware::Exporter, path: metrics.endpoint
+        use Vmpooler::Metrics::Promstats::CollectorMiddleware, metrics_prefix: "#{metrics.prometheus_prefix}_http"
+        use Prometheus::Middleware::Exporter, path: metrics.prometheus_endpoint
       end
 
       if torun.include? :api

data/lib/vmpooler/api/v1.rb

@@ -89,18 +89,16 @@ module Vmpooler
         template_backends += aliases
         weighted_pools = get_pool_weights(template_backends)
 
-        pickup = Pickup.new(weighted_pools) if weighted_pools.count == template_backends.count
-        count.to_i.times do
-          if pickup
+        if weighted_pools.count > 1 && weighted_pools.count == template_backends.count
+          pickup = Pickup.new(weighted_pools)
+          count.to_i.times do
             selection << pickup.pick
-          else
+          end
+        else
+          count.to_i.times do
             selection << template_backends.sample
           end
         end
-      else
-        count.to_i.times do
-          selection << template
-        end
       end
 
       count_selection(selection)
@@ -809,7 +807,7 @@ module Vmpooler
 
     post "#{api_prefix}/ondemandvm/?" do
       content_type :json
-      metrics.increment('api_vm.post.ondemand.requestid')
+      metrics.increment('http_requests_vm_total.post.ondemand.requestid')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 
@@ -847,7 +845,7 @@ module Vmpooler
     post "#{api_prefix}/ondemandvm/:template/?" do
       content_type :json
       result = { 'ok' => false }
-      metrics.increment('api_vm.delete.ondemand.template')
+      metrics.increment('http_requests_vm_total.delete.ondemand.template')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 
@@ -874,7 +872,7 @@ module Vmpooler
 
     get "#{api_prefix}/ondemandvm/:requestid/?" do
       content_type :json
-      metrics.increment('api_vm.get.ondemand.request')
+      metrics.increment('http_requests_vm_total.get.ondemand.request')
 
       status 404
       result = check_ondemand_request(params[:requestid])
@@ -885,7 +883,7 @@ module Vmpooler
     delete "#{api_prefix}/ondemandvm/:requestid/?" do
       content_type :json
       need_token! if Vmpooler::API.settings.config[:auth]
-      metrics.increment('api_vm.delete.ondemand.request')
+      metrics.increment('http_requests_vm_total.delete.ondemand.request')
 
       status 404
       result = delete_ondemand_request(params[:requestid])
@@ -896,7 +894,7 @@ module Vmpooler
     post "#{api_prefix}/vm/?" do
       content_type :json
       result = { 'ok' => false }
-      metrics.increment('api_vm.post.vm.checkout')
+      metrics.increment('http_requests_vm_total.post.vm.checkout')
 
       payload = JSON.parse(request.body.read)
 
@@ -1051,7 +1049,7 @@ module Vmpooler
     post "#{api_prefix}/vm/:template/?" do
       content_type :json
       result = { 'ok' => false }
-      metrics.increment('api_vm.get.vm.template')
+      metrics.increment('http_requests_vm_total.get.vm.template')
 
       payload = extract_templates_from_query_params(params[:template])
 
@@ -1075,7 +1073,7 @@ module Vmpooler
 
     get "#{api_prefix}/vm/:hostname/?" do
       content_type :json
-      metrics.increment('api_vm.get.vm.hostname')
+      metrics.increment('http_requests_vm_total.get.vm.hostname')
 
       result = {}
 
@@ -1148,7 +1146,7 @@ module Vmpooler
 
     delete "#{api_prefix}/vm/:hostname/?" do
       content_type :json
-      metrics.increment('api_vm.delete.vm.hostname')
+      metrics.increment('http_requests_vm_total.delete.vm.hostname')
 
       result = {}
 
@@ -1177,7 +1175,7 @@ module Vmpooler
 
     put "#{api_prefix}/vm/:hostname/?" do
       content_type :json
-      metrics.increment('api_vm.put.vm.modify')
+      metrics.increment('http_requests_vm_total.put.vm.modify')
 
       status 404
       result = { 'ok' => false }
@@ -1254,7 +1252,7 @@ module Vmpooler
 
     post "#{api_prefix}/vm/:hostname/disk/:size/?" do
       content_type :json
-      metrics.increment('api_vm.post.vm.disksize')
+      metrics.increment('http_requests_vm_total.post.vm.disksize')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 
@@ -1278,7 +1276,7 @@ module Vmpooler
 
     post "#{api_prefix}/vm/:hostname/snapshot/?" do
       content_type :json
-      metrics.increment('api_vm.post.vm.snapshot')
+      metrics.increment('http_requests_vm_total.post.vm.snapshot')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 
@@ -1304,7 +1302,7 @@ module Vmpooler
 
     post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
       content_type :json
-      metrics.increment('api_vm.post.vm.disksize')
+      metrics.increment('http_requests_vm_total.post.vm.disksize')
 
       need_token! if Vmpooler::API.settings.config[:auth]
 

data/lib/vmpooler/metrics/promstats.rb

@@ -5,7 +5,7 @@ require 'prometheus/client'
 module Vmpooler
   class Metrics
     class Promstats < Metrics
-      attr_reader :prefix, :endpoint, :metrics_prefix
+      attr_reader :prefix, :prometheus_endpoint, :prometheus_prefix
 
       # Constants for Metric Types
       M_COUNTER   = 1
@@ -21,25 +21,139 @@ module Vmpooler
       REDIS_CONNECT_BUCKETS = [1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 18.0, 23.0].freeze
 
       @p_metrics = {}
+      @torun = []
 
       def initialize(logger, params = {})
         @prefix = params['prefix'] || 'vmpooler'
-        @metrics_prefix = params['metrics_prefix'] || 'vmpooler'
-        @endpoint = params['endpoint'] || '/prometheus'
+        @prometheus_prefix = params['prometheus_prefix'] || 'vmpooler'
+        @prometheus_endpoint = params['prometheus_endpoint'] || '/prometheus'
         @logger = logger
 
         # Setup up prometheus registry and data structures
         @prometheus = Prometheus::Client.registry
       end
 
-      # Metrics structure used to register the metrics and also translate/interpret the incoming metrics.
+=begin # rubocop:disable Style/BlockComments
+      The Metrics table is used to register metrics and translate/interpret the incoming metrics.
+
+      This table describes all of the prometheus metrics that are recognised by the application.
+      The background documentation for defining metrics is at: https://prometheus.io/docs/introduction/
+      In particular, the naming practices should be adhered to: https://prometheus.io/docs/practices/naming/
+      The Ruby Client docs are also useful: https://github.com/prometheus/client_ruby
+
+      The table here allows the currently used stats definitions to be translated correctly for Prometheus.
+      The current format is of the form A.B.C, where the final fields may be actual values (e.g. poolname).
+      Prometheus metrics cannot use the '.' as a character, so this is either translated into '_' or
+      variable parameters are expressed as labels accompanying the metric.
+
+      Sample statistics are:
+          # Example showing hostnames (FQDN)
+          migrate_from.pix-jj26-chassis1-2.ops.puppetlabs.net
+          migrate_to.pix-jj26-chassis1-8.ops.puppetlabs.net
+
+          # Example showing poolname as a parameter
+          poolreset.invalid.centos-8-x86_64
+
+          # Examples showing similar sub-typed checkout stats
+          checkout.empty.centos-8-x86_64
+          checkout.invalid.centos-8-x86_64
+          checkout.invalid.unknown
+          checkout.success.centos-8-x86_64
+
+          # Stats without any final parameter.
+          connect.fail
+          connect.open
+          delete.failed
+          delete.success
+
+          # Stats with multiple param_labels
+          vmpooler_user.debian-8-x86_64-pixa4.john
+
+        The metrics implementation here preserves the existing framework which will continue to support
+        graphite and statsd (since vmpooler is used outside of puppet). Some rationalisation and renaming
+        of the actual metrics was done to get a more usable model to fit within the prometheus framework.
+        This particularly applies to the user stats collected once individual machines are terminated as
+        this would have challenged prometheus' ability due to multiple (8) parameters being collected
+        in a single measure (which has a very high cardinality).
+
+        Prometheus requires all metrics to be pre-registered (which is the primary reason for this
+        table) and also uses labels to differentiate the characteristics of the measurement. This
+        is used throughout to capture information such as poolnames. So for example, this is a sample
+        of the prometheus metrics generated for the "vmpooler_ready" measurement:
+
+          # TYPE vmpooler_ready gauge
+          # HELP vmpooler_ready vmpooler number of machines in ready State
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="win-10-ent-x86_64-pixa4"} 2.0
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="debian-8-x86_64-pixa4"} 2.0
+          vmpooler_ready{vmpooler_instance="vmpooler",poolname="centos-8-x86_64-pixa4"} 2.0
+
+        Prometheus supports the following metric types:
+        (see https://prometheus.io/docs/concepts/metric_types/)
+
+          Counter (increment):
+            A counter is a cumulative metric that represents a single monotonically increasing counter whose
+            value can only increase or be reset to zero on restart
+
+          Gauge:
+            A gauge is a metric that represents a single numerical value that can arbitrarily go up and down.
+
+          Histogram:
+            A histogram samples observations (usually things like request durations or response sizes) and
+            counts them in configurable buckets. It also provides a sum of all observed values.
+            This replaces the timer metric supported by statsd
+
+          Summary :
+            Summary provides a total count of observations and a sum of all observed values, it calculates
+            configurable quantiles over a sliding time window.
+            (Summary is not used in vmpooler)
+
+        vmpooler_metrics_table is a table of hashes, where the hash key represents the first part of the
+        metric name, e.g. for the metric 'delete.*' (see above) the key would be 'delete:'. "Sub-metrics",
+        are supported, again for the 'delete.*' example, this can be subbed into '.failed' and '.success'
+
+        The entries within the hash as are follows:
+
+          mtype:
+            Metric type, which is one of the following constants:
+              M_COUNTER   = 1
+              M_GAUGE     = 2
+              M_SUMMARY   = 3
+              M_HISTOGRAM = 4
+
+          torun:
+            Indicates which process the metric is for - within vmpooler this is either ':api' or ':manager'
+            (there is a suggestion that we change this to two separate tables).
+
+          docstring:
+            Documentation string for the metric - this is displayed as HELP text by the endpoint.
+
+          metric_suffixes:
+            Array of sub-metrics of the form 'sub-metric: "doc-string for sub-metric"'. This supports
+            the generation of individual sub-metrics for all elements in the array.
+
+          param_labels:
+            This is an optional array of symbols for the final labels in a metric. It should not be
+            specified if there are no additional parameters.
+
+            If it specified, it can either be a single symbol, or two or more symbols. The treatment
+            differs if there is only one symbol given as all of the remainder of the metric string
+            supplied is collected into a label with the symbol name. This allows the handling of
+            node names (FQDN).
+
+            To illustrate:
+            1. In the 'connect.*' or 'delete.*' example above, it should not be specified.
+            2. For the 'migrate_from.*' example above, the remainder of the measure is collected
+               as the 'host_name' label.
+            3. For the 'vmpooler_user' example above, the first parameter is treated as the pool
+               name, and the second as the username.
+
+=end
       def vmpooler_metrics_table
         {
           errors: {
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'Count of errors for pool',
-            prom_metric_prefix: "#{@metrics_prefix}_errors",
             metric_suffixes: {
               markedasfailed: 'timeout waiting for instance to initialise',
               duplicatehostname: 'unable to create instance due to duplicate hostname',
@@ -51,42 +165,36 @@ module Vmpooler
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'Number of pool instances this user created created',
-            prom_metric_prefix: "#{@metrics_prefix}_user",
             param_labels: %i[user poolname]
           },
           usage_litmus: {
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'Pools by Litmus job usage',
-            prom_metric_prefix: "#{@metrics_prefix}_usage_litmus",
             param_labels: %i[user poolname]
           },
           usage_jenkins_instance: {
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'Pools by Jenkins instance usage',
-            prom_metric_prefix: "#{@metrics_prefix}_usage_jenkins_instance",
             param_labels: %i[jenkins_instance value_stream poolname]
           },
           usage_branch_project: {
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'Pools by branch/project usage',
-            prom_metric_prefix: "#{@metrics_prefix}_usage_branch_project",
             param_labels: %i[branch project poolname]
           },
           usage_job_component: {
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'Pools by job/component usage',
-            prom_metric_prefix: "#{@metrics_prefix}_usage_job_component",
             param_labels: %i[job_name component_to_test poolname]
           },
           checkout: {
             mtype: M_COUNTER,
             torun: %i[api],
             docstring: 'Pool checkout counts',
-            prom_metric_prefix: "#{@metrics_prefix}_checkout",
             metric_suffixes: {
               nonresponsive: 'checkout failed - non responsive machine',
               empty: 'checkout failed - no machine',
@@ -99,7 +207,6 @@ module Vmpooler
             mtype: M_COUNTER,
             torun: %i[api],
             docstring: 'Delete machine',
-            prom_metric_prefix: "#{@metrics_prefix}_delete",
             metric_suffixes: {
               success: 'succeeded',
               failed: 'failed'
@@ -110,7 +217,6 @@ module Vmpooler
             mtype: M_COUNTER,
             torun: %i[api],
             docstring: 'Ondemand request',
-            prom_metric_prefix: "#{@metrics_prefix}_ondemandrequest_generate",
             metric_suffixes: {
               duplicaterequests: 'failed duplicate request',
               success: 'succeeded'
@@ -121,7 +227,6 @@ module Vmpooler
             mtype: M_COUNTER,
             torun: %i[api],
             docstring: 'Ondemand request failure',
-            prom_metric_prefix: "#{@metrics_prefix}_ondemandrequest_fail",
             metric_suffixes: {
               toomanyrequests: 'too many requests',
               invalid: 'invalid poolname'
@@ -132,7 +237,6 @@ module Vmpooler
             mtype: M_COUNTER,
             torun: %i[api],
             docstring: 'vmpooler pool configuration request',
-            prom_metric_prefix: "#{@metrics_prefix}_config",
             metric_suffixes: { invalid: 'Invalid' },
             param_labels: %i[poolname]
           },
@@ -140,7 +244,6 @@ module Vmpooler
             mtype: M_COUNTER,
             torun: %i[api],
             docstring: 'Pool reset counter',
-            prom_metric_prefix: "#{@metrics_prefix}_poolreset",
             metric_suffixes: { invalid: 'Invalid Pool' },
             param_labels: %i[poolname]
           },
@@ -148,7 +251,6 @@ module Vmpooler
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'vmpooler connect (to vSphere)',
-            prom_metric_prefix: "#{@metrics_prefix}_connect",
             metric_suffixes: {
               open: 'Connect Succeeded',
               fail: 'Connect Failed'
@@ -159,42 +261,36 @@ module Vmpooler
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'vmpooler machine migrated from',
-            prom_metric_prefix: "#{@metrics_prefix}_migrate_from",
             param_labels: %i[host_name]
           },
           migrate_to: {
             mtype: M_COUNTER,
             torun: %i[manager],
             docstring: 'vmpooler machine migrated to',
-            prom_metric_prefix: "#{@metrics_prefix}_migrate_to",
             param_labels: %i[host_name]
           },
-          api_vm: {
+          http_requests_vm_total: {
             mtype: M_COUNTER,
             torun: %i[api],
             docstring: 'Total number of HTTP request/sub-operations handled by the Rack application under the /vm endpoint',
-            prom_metric_prefix: "#{@metrics_prefix}_http_requests_vm_total",
             param_labels: %i[method subpath operation]
           },
           ready: {
             mtype: M_GAUGE,
             torun: %i[manager],
             docstring: 'vmpooler number of machines in ready State',
-            prom_metric_prefix: "#{@metrics_prefix}_ready",
             param_labels: %i[poolname]
           },
           running: {
             mtype: M_GAUGE,
             torun: %i[manager],
             docstring: 'vmpooler number of machines running',
-            prom_metric_prefix: "#{@metrics_prefix}_running",
             param_labels: %i[poolname]
           },
           connection_available: {
             mtype: M_GAUGE,
             torun: %i[manager],
             docstring: 'vmpooler redis connections available',
-            prom_metric_prefix: "#{@metrics_prefix}_connection_available",
             param_labels: %i[type provider]
           },
           time_to_ready_state: {
@@ -202,7 +298,6 @@ module Vmpooler
             torun: %i[manager],
             buckets: POOLER_READY_TIME_BUCKETS,
             docstring: 'Time taken for machine to read ready state for pool',
-            prom_metric_prefix: "#{@metrics_prefix}_time_to_ready_state",
             param_labels: %i[poolname]
           },
           migrate: {
@@ -210,7 +305,6 @@ module Vmpooler
             torun: %i[manager],
             buckets: POOLER_CLONE_TIME_BUCKETS,
             docstring: 'vmpooler time taken to migrate machine for pool',
-            prom_metric_prefix: "#{@metrics_prefix}_migrate",
             param_labels: %i[poolname]
           },
           clone: {
@@ -218,7 +312,6 @@ module Vmpooler
             torun: %i[manager],
             buckets: POOLER_CLONE_TIME_BUCKETS,
             docstring: 'vmpooler time taken to clone machine',
-            prom_metric_prefix: "#{@metrics_prefix}_clone",
             param_labels: %i[poolname]
           },
           destroy: {
@@ -226,7 +319,6 @@ module Vmpooler
             torun: %i[manager],
             buckets: POOLER_CLONE_TIME_BUCKETS,
             docstring: 'vmpooler time taken to destroy machine',
-            prom_metric_prefix: "#{@metrics_prefix}_destroy",
             param_labels: %i[poolname]
           },
           connection_waited: {
@@ -234,7 +326,6 @@ module Vmpooler
             torun: %i[manager],
             buckets: REDIS_CONNECT_BUCKETS,
             docstring: 'vmpooler redis connection wait time',
-            prom_metric_prefix: "#{@metrics_prefix}_connection_waited",
             param_labels: %i[type provider]
           }
         }
@@ -278,8 +369,9 @@ module Vmpooler
       # Top level method to register all the prometheus metrics.
 
       def setup_prometheus_metrics(torun)
+        @torun = torun
         @p_metrics = vmpooler_metrics_table
-        @p_metrics.each do |_name, metric_spec|
+        @p_metrics.each do |name, metric_spec|
           # Only register metrics appropriate to api or manager
           next if (torun & metric_spec[:torun]).empty?
 
@@ -288,7 +380,7 @@ module Vmpooler
             metric_spec[:metric_suffixes].each do |metric_suffix|
               add_prometheus_metric(
                 metric_spec,
-                "#{metric_spec[:prom_metric_prefix]}_#{metric_suffix[0]}",
+                "#{@prometheus_prefix}_#{name}_#{metric_suffix[0]}",
                 "#{metric_spec[:docstring]} #{metric_suffix[1]}"
               )
             end
@@ -296,7 +388,7 @@ module Vmpooler
             # No Additional counter suffixes so register this as metric.
             add_prometheus_metric(
               metric_spec,
-              metric_spec[:prom_metric_prefix],
+              "#{@prometheus_prefix}_#{name}",
               metric_spec[:docstring]
             )
           end
@@ -309,15 +401,18 @@ module Vmpooler
         metric_key = sublabels.shift.to_sym
         raise("Invalid Metric #{metric_key} for #{label}") unless @p_metrics.key? metric_key
 
-        metric = @p_metrics[metric_key].clone
+        metric_spec = @p_metrics[metric_key]
+        raise("Invalid Component #{component} for #{metric_key}") if (metric_spec[:torun] & @torun).nil?
+
+        metric = metric_spec.clone
 
         if metric.key? :metric_suffixes
           metric_subkey = sublabels.shift.to_sym
           raise("Invalid Metric #{metric_key}_#{metric_subkey} for #{label}") unless metric[:metric_suffixes].key? metric_subkey.to_sym
 
-          metric[:metric_name] = "#{metric[:prom_metric_prefix]}_#{metric_subkey}"
+          metric[:metric_name] = "#{@prometheus_prefix}_#{metric_key}_#{metric_subkey}"
         else
-          metric[:metric_name] = metric[:prom_metric_prefix]
+          metric[:metric_name] = "#{@prometheus_prefix}_#{metric_key}"
         end
 
         # Check if we are looking for a parameter value at last element.

data/lib/vmpooler/metrics/promstats/collector_middleware.rb

@@ -114,6 +114,9 @@ module Vmpooler
           path
             .gsub(%r{/vm/.+$}, '/vm')
             .gsub(%r{/ondemand/.+$}, '/ondemand')
+            .gsub(%r{/token/.+$}, '/token')
+            .gsub(%r{/lib/.+$}, '/lib')
+            .gsub(%r{/img/.+$}, '/img')
         end
       end
     end

data/lib/vmpooler/pool_manager.rb

@@ -368,7 +368,7 @@ module Vmpooler
           $metrics.increment("errors.duplicatehostname.#{pool_name}")
           $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
         elsif !dns_available
-          $metrics.increment("errors.staledns.#{hostname}")
+          $metrics.increment("errors.staledns.#{pool_name}")
           $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS")
         end
       end

data/lib/vmpooler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vmpooler
-  VERSION = '0.14.2'
+  VERSION = '0.14.7'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vmpooler
 version: !ruby/object:Gem::Version
-  version: 0.14.2
+  version: 0.14.7
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-03 00:00:00.000000000 Z
+date: 2020-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pickup
@@ -84,16 +84,22 @@ dependencies:
   name: rbvmomi
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement