vmpooler 0.13.3 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fe0ef9bcf3e665cc952fce0975a4381883fca200ef93a883e54c7db7e1fc10e7
4
- data.tar.gz: d44b61c41a57f940cf0df4bf07aabed10b1a72e91cae5b2105f092a7d98d6d05
3
+ metadata.gz: 74df51cf1d525f0bc7afe2da930e68d0b2e1e014e10d184213556e4a13bd170b
4
+ data.tar.gz: 7cfe6d26ebb99e421f2b9daeabe1dfcb610cc8457cae1f79c5632df33eb6d426
5
5
  SHA512:
6
- metadata.gz: e0e7bbec11ceecc5fdee0fe15b3d37e290a6c884a708a6a5acb2e790d54fb1f1e24ec6966973f02bbecd204a29497e9c17ae3b8be26a0d3f15cec508a0d64532
7
- data.tar.gz: 7302cf41676a65ff5d0bb4108a8f5ac03a96a92e3cd0ec56deee2f2a60d6e73f09eee569ef0f4140e3c27e81a16c3e2d24b951764a772bdeaa082a96d8bde1fe
6
+ metadata.gz: d4e00e1cf43c08652b6186a741f048a487c1300744a2a40912759bdf06a2ad306fd59ae9adb10364be191047edc4cc84d8ea99fbf211806d59228b0993e61d0f
7
+ data.tar.gz: 4948f3405e5969bcd55b350604d1a2b8bfa9270960b635c4448a9478764f79bef1b2af101e11a3389587cdc2bd0fb22f28c1b7d1582227414aa42764d8451a74
@@ -11,33 +11,38 @@ redis_connection_pool_size = config[:redis]['connection_pool_size']
11
11
  redis_connection_pool_timeout = config[:redis]['connection_pool_timeout']
12
12
  logger_file = config[:config]['logfile']
13
13
 
14
- metrics = Vmpooler.new_metrics(config)
14
+ logger = Vmpooler::Logger.new logger_file
15
+ metrics = Vmpooler::Metrics.init(logger, config)
15
16
 
16
17
  torun_threads = []
17
18
  if ARGV.count == 0
18
- torun = ['api', 'manager']
19
+ torun = %i[api manager]
19
20
  else
20
21
  torun = []
21
- torun << 'api' if ARGV.include? 'api'
22
- torun << 'manager' if ARGV.include? 'manager'
22
+ torun << :api if ARGV.include? 'api'
23
+ torun << :manager if ARGV.include? 'manager'
23
24
  exit(2) if torun.empty?
24
25
  end
25
26
 
26
- if torun.include? 'api'
27
+ if torun.include? :api
27
28
  api = Thread.new do
28
- thr = Vmpooler::API.new
29
29
  redis = Vmpooler.new_redis(redis_host, redis_port, redis_password)
30
- thr.helpers.configure(config, redis, metrics)
31
- thr.helpers.execute!
30
+ Vmpooler::API.execute(torun, config, redis, metrics, logger)
32
31
  end
33
32
  torun_threads << api
33
+ elsif metrics.respond_to?(:setup_prometheus_metrics)
34
+ # Run the cut down API - Prometheus Metrics only.
35
+ prometheus_only_api = Thread.new do
36
+ Vmpooler::API.execute(torun, config, nil, metrics, logger)
37
+ end
38
+ torun_threads << prometheus_only_api
34
39
  end
35
40
 
36
- if torun.include? 'manager'
41
+ if torun.include? :manager
37
42
  manager = Thread.new do
38
43
  Vmpooler::PoolManager.new(
39
44
  config,
40
- Vmpooler.new_logger(logger_file),
45
+ logger,
41
46
  Vmpooler.redis_connection_pool(redis_host, redis_port, redis_password, redis_connection_pool_size, redis_connection_pool_timeout, metrics),
42
47
  metrics
43
48
  ).execute!
@@ -15,7 +15,7 @@ module Vmpooler
15
15
  require 'timeout'
16
16
  require 'yaml'
17
17
 
18
- %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
18
+ %w[api metrics logger pool_manager generic_connection_pool].each do |lib|
19
19
  require "vmpooler/#{lib}"
20
20
  end
21
21
 
@@ -84,6 +84,7 @@ module Vmpooler
84
84
  parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
85
85
  parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
86
86
  parsed_config[:config]['usage_stats'] = ENV['USAGE_STATS'] if ENV['USAGE_STATS']
87
+ parsed_config[:config]['request_logger'] = ENV['REQUEST_LOGGER'] if ENV['REQUEST_LOGGER']
87
88
 
88
89
  parsed_config[:redis] = parsed_config[:redis] || {}
89
90
  parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
@@ -166,7 +167,8 @@ module Vmpooler
166
167
  def self.redis_connection_pool(host, port, password, size, timeout, metrics)
167
168
  Vmpooler::PoolManager::GenericConnectionPool.new(
168
169
  metrics: metrics,
169
- metric_prefix: 'redis_connection_pool',
170
+ connpool_type: 'redis_connection_pool',
171
+ connpool_provider: 'manager',
170
172
  size: size,
171
173
  timeout: timeout
172
174
  ) do
@@ -180,20 +182,6 @@ module Vmpooler
180
182
  Redis.new(host: host, port: port, password: password)
181
183
  end
182
184
 
183
- def self.new_logger(logfile)
184
- Vmpooler::Logger.new logfile
185
- end
186
-
187
- def self.new_metrics(params)
188
- if params[:statsd]
189
- Vmpooler::Statsd.new(params[:statsd])
190
- elsif params[:graphite]
191
- Vmpooler::Graphite.new(params[:graphite])
192
- else
193
- Vmpooler::DummyStatsd.new
194
- end
195
- end
196
-
197
185
  def self.pools(conf)
198
186
  conf[:pools]
199
187
  end
@@ -2,51 +2,58 @@
2
2
 
3
3
  module Vmpooler
4
4
  class API < Sinatra::Base
5
- def initialize
6
- super
5
+ # Load API components
6
+ %w[helpers dashboard reroute v1 request_logger].each do |lib|
7
+ require "vmpooler/api/#{lib}"
7
8
  end
9
+ # Load dashboard components
10
+ require 'vmpooler/dashboard'
8
11
 
9
- not_found do
10
- content_type :json
12
+ def self.execute(torun, config, redis, metrics, logger)
13
+ self.settings.set :config, config
14
+ self.settings.set :redis, redis unless redis.nil?
15
+ self.settings.set :metrics, metrics
16
+ self.settings.set :checkoutlock, Mutex.new
11
17
 
12
- result = {
13
- ok: false
14
- }
18
+ # Deflating in all situations
19
+ # https://www.schneems.com/2017/11/08/80-smaller-rails-footprint-with-rack-deflate/
20
+ use Rack::Deflater
15
21
 
16
- JSON.pretty_generate(result)
17
- end
22
+ # not_found clause placed here to fix rspec test issue.
23
+ not_found do
24
+ content_type :json
18
25
 
19
- # Load dashboard components
20
- begin
21
- require 'dashboard'
22
- rescue LoadError
23
- require File.expand_path(File.join(File.dirname(__FILE__), 'dashboard'))
24
- end
26
+ result = {
27
+ ok: false
28
+ }
25
29
 
26
- use Vmpooler::Dashboard
30
+ JSON.pretty_generate(result)
31
+ end
27
32
 
28
- # Load API components
29
- %w[helpers dashboard reroute v1].each do |lib|
30
- begin
31
- require "api/#{lib}"
32
- rescue LoadError
33
- require File.expand_path(File.join(File.dirname(__FILE__), 'api', lib))
33
+ if metrics.respond_to?(:setup_prometheus_metrics)
34
+ # Prometheus metrics are only setup if actually specified
35
+ # in the config file.
36
+ metrics.setup_prometheus_metrics(torun)
37
+
38
+ # Using customised collector that filters out hostnames on API paths
39
+ require 'vmpooler/metrics/promstats/collector_middleware'
40
+ require 'prometheus/middleware/exporter'
41
+ use Vmpooler::Metrics::Promstats::CollectorMiddleware, metrics_prefix: "#{metrics.metrics_prefix}_http"
42
+ use Prometheus::Middleware::Exporter, path: metrics.endpoint
34
43
  end
35
- end
36
44
 
37
- use Vmpooler::API::Dashboard
38
- use Vmpooler::API::Reroute
39
- use Vmpooler::API::V1
45
+ if torun.include? :api
46
+ # Enable API request logging only if required
47
+ use Vmpooler::API::RequestLogger, logger: logger if config[:config]['request_logger']
40
48
 
41
- def configure(config, redis, metrics)
42
- self.settings.set :config, config
43
- self.settings.set :redis, redis
44
- self.settings.set :metrics, metrics
45
- self.settings.set :checkoutlock, Mutex.new
46
- end
49
+ use Vmpooler::Dashboard
50
+ use Vmpooler::API::Dashboard
51
+ use Vmpooler::API::Reroute
52
+ use Vmpooler::API::V1
53
+ end
47
54
 
48
- def execute!
49
- self.settings.run!
55
+ # Get thee started O WebServer
56
+ self.run!
50
57
  end
51
58
  end
52
59
  end
@@ -13,7 +13,7 @@ module Vmpooler
13
13
  def valid_token?(backend)
14
14
  return false unless has_token?
15
15
 
16
- backend.exists('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN']) ? true : false
16
+ backend.exists?('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN']) ? true : false
17
17
  end
18
18
 
19
19
  def validate_token(backend)
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Vmpooler
4
+ class API
5
+ class RequestLogger
6
+ attr_reader :app
7
+
8
+ def initialize(app, options = {})
9
+ @app = app
10
+ @logger = options[:logger]
11
+ end
12
+
13
+ def call(env)
14
+ status, headers, body = @app.call(env)
15
+ @logger.log('s', "[ ] API: Method: #{env['REQUEST_METHOD']}, Status: #{status}, Path: #{env['PATH_INFO']}, Body: #{body}")
16
+ [status, headers, body]
17
+ end
18
+ end
19
+ end
20
+ end
@@ -339,7 +339,7 @@ module Vmpooler
339
339
  payload&.each do |poolname, count|
340
340
  next unless count.to_i > config['max_ondemand_instances_per_request']
341
341
 
342
- metrics.increment('ondemandrequest.toomanyrequests.' + poolname)
342
+ metrics.increment('ondemandrequest_fail.toomanyrequests.' + poolname)
343
343
  return true
344
344
  end
345
345
  false
@@ -360,10 +360,10 @@ module Vmpooler
360
360
  request_id ||= generate_request_id
361
361
  result['request_id'] = request_id
362
362
 
363
- if backend.exists("vmpooler__odrequest__#{request_id}")
363
+ if backend.exists?("vmpooler__odrequest__#{request_id}")
364
364
  result['message'] = "request_id '#{request_id}' has already been created"
365
365
  status 409
366
- metrics.increment('ondemandrequest.generate.duplicaterequests')
366
+ metrics.increment('ondemandrequest_generate.duplicaterequests')
367
367
  return result
368
368
  end
369
369
 
@@ -387,7 +387,7 @@ module Vmpooler
387
387
 
388
388
  result['domain'] = config['domain'] if config['domain']
389
389
  result[:ok] = true
390
- metrics.increment('ondemandrequest.generate.success')
390
+ metrics.increment('ondemandrequest_generate.success')
391
391
  result
392
392
  end
393
393
 
@@ -809,6 +809,7 @@ module Vmpooler
809
809
 
810
810
  post "#{api_prefix}/ondemandvm/?" do
811
811
  content_type :json
812
+ metrics.increment('api_vm.post.ondemand.requestid')
812
813
 
813
814
  need_token! if Vmpooler::API.settings.config[:auth]
814
815
 
@@ -824,12 +825,12 @@ module Vmpooler
824
825
  else
825
826
  result[:bad_templates] = invalid
826
827
  invalid.each do |bad_template|
827
- metrics.increment('ondemandrequest.invalid.' + bad_template)
828
+ metrics.increment('ondemandrequest_fail.invalid.' + bad_template)
828
829
  end
829
830
  status 404
830
831
  end
831
832
  else
832
- metrics.increment('ondemandrequest.invalid.unknown')
833
+ metrics.increment('ondemandrequest_fail.invalid.unknown')
833
834
  status 404
834
835
  end
835
836
  rescue JSON::ParserError
@@ -846,6 +847,7 @@ module Vmpooler
846
847
  post "#{api_prefix}/ondemandvm/:template/?" do
847
848
  content_type :json
848
849
  result = { 'ok' => false }
850
+ metrics.increment('api_vm.delete.ondemand.template')
849
851
 
850
852
  need_token! if Vmpooler::API.settings.config[:auth]
851
853
 
@@ -858,12 +860,12 @@ module Vmpooler
858
860
  else
859
861
  result[:bad_templates] = invalid
860
862
  invalid.each do |bad_template|
861
- metrics.increment('ondemandrequest.invalid.' + bad_template)
863
+ metrics.increment('ondemandrequest_fail.invalid.' + bad_template)
862
864
  end
863
865
  status 404
864
866
  end
865
867
  else
866
- metrics.increment('ondemandrequest.invalid.unknown')
868
+ metrics.increment('ondemandrequest_fail.invalid.unknown')
867
869
  status 404
868
870
  end
869
871
 
@@ -872,6 +874,7 @@ module Vmpooler
872
874
 
873
875
  get "#{api_prefix}/ondemandvm/:requestid/?" do
874
876
  content_type :json
877
+ metrics.increment('api_vm.get.ondemand.request')
875
878
 
876
879
  status 404
877
880
  result = check_ondemand_request(params[:requestid])
@@ -882,6 +885,7 @@ module Vmpooler
882
885
  delete "#{api_prefix}/ondemandvm/:requestid/?" do
883
886
  content_type :json
884
887
  need_token! if Vmpooler::API.settings.config[:auth]
888
+ metrics.increment('api_vm.delete.ondemand.request')
885
889
 
886
890
  status 404
887
891
  result = delete_ondemand_request(params[:requestid])
@@ -892,6 +896,7 @@ module Vmpooler
892
896
  post "#{api_prefix}/vm/?" do
893
897
  content_type :json
894
898
  result = { 'ok' => false }
899
+ metrics.increment('api_vm.post.vm.checkout')
895
900
 
896
901
  payload = JSON.parse(request.body.read)
897
902
 
@@ -1034,6 +1039,7 @@ module Vmpooler
1034
1039
  post "#{api_prefix}/vm/:template/?" do
1035
1040
  content_type :json
1036
1041
  result = { 'ok' => false }
1042
+ metrics.increment('api_vm.get.vm.template')
1037
1043
 
1038
1044
  payload = extract_templates_from_query_params(params[:template])
1039
1045
 
@@ -1057,6 +1063,7 @@ module Vmpooler
1057
1063
 
1058
1064
  get "#{api_prefix}/vm/:hostname/?" do
1059
1065
  content_type :json
1066
+ metrics.increment('api_vm.get.vm.hostname')
1060
1067
 
1061
1068
  result = {}
1062
1069
 
@@ -1129,6 +1136,7 @@ module Vmpooler
1129
1136
 
1130
1137
  delete "#{api_prefix}/vm/:hostname/?" do
1131
1138
  content_type :json
1139
+ metrics.increment('api_vm.delete.vm.hostname')
1132
1140
 
1133
1141
  result = {}
1134
1142
 
@@ -1146,8 +1154,9 @@ module Vmpooler
1146
1154
 
1147
1155
  status 200
1148
1156
  result['ok'] = true
1157
+ metrics.increment('delete.success')
1149
1158
  else
1150
- metrics.increment('delete.srem.failed')
1159
+ metrics.increment('delete.failed')
1151
1160
  end
1152
1161
  end
1153
1162
 
@@ -1156,6 +1165,7 @@ module Vmpooler
1156
1165
 
1157
1166
  put "#{api_prefix}/vm/:hostname/?" do
1158
1167
  content_type :json
1168
+ metrics.increment('api_vm.put.vm.modify')
1159
1169
 
1160
1170
  status 404
1161
1171
  result = { 'ok' => false }
@@ -1164,7 +1174,7 @@ module Vmpooler
1164
1174
 
1165
1175
  params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
1166
1176
 
1167
- if backend.exists('vmpooler__vm__' + params[:hostname])
1177
+ if backend.exists?('vmpooler__vm__' + params[:hostname])
1168
1178
  begin
1169
1179
  jdata = JSON.parse(request.body.read)
1170
1180
  rescue StandardError
@@ -1232,6 +1242,7 @@ module Vmpooler
1232
1242
 
1233
1243
  post "#{api_prefix}/vm/:hostname/disk/:size/?" do
1234
1244
  content_type :json
1245
+ metrics.increment('api_vm.post.vm.disksize')
1235
1246
 
1236
1247
  need_token! if Vmpooler::API.settings.config[:auth]
1237
1248
 
@@ -1240,7 +1251,7 @@ module Vmpooler
1240
1251
 
1241
1252
  params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
1242
1253
 
1243
- if ((params[:size].to_i > 0 )and (backend.exists('vmpooler__vm__' + params[:hostname])))
1254
+ if ((params[:size].to_i > 0 )and (backend.exists?('vmpooler__vm__' + params[:hostname])))
1244
1255
  result[params[:hostname]] = {}
1245
1256
  result[params[:hostname]]['disk'] = "+#{params[:size]}gb"
1246
1257
 
@@ -1255,6 +1266,7 @@ module Vmpooler
1255
1266
 
1256
1267
  post "#{api_prefix}/vm/:hostname/snapshot/?" do
1257
1268
  content_type :json
1269
+ metrics.increment('api_vm.post.vm.snapshot')
1258
1270
 
1259
1271
  need_token! if Vmpooler::API.settings.config[:auth]
1260
1272
 
@@ -1263,7 +1275,7 @@ module Vmpooler
1263
1275
 
1264
1276
  params[:hostname] = hostname_shorten(params[:hostname], config['domain'])
1265
1277
 
1266
- if backend.exists('vmpooler__vm__' + params[:hostname])
1278
+ if backend.exists?('vmpooler__vm__' + params[:hostname])
1267
1279
  result[params[:hostname]] = {}
1268
1280
 
1269
1281
  o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
@@ -1280,6 +1292,7 @@ module Vmpooler
1280
1292
 
1281
1293
  post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
1282
1294
  content_type :json
1295
+ metrics.increment('api_vm.post.vm.disksize')
1283
1296
 
1284
1297
  need_token! if Vmpooler::API.settings.config[:auth]
1285
1298
 
@@ -11,8 +11,10 @@ module Vmpooler
11
11
  def initialize(options = {}, &block)
12
12
  super(options, &block)
13
13
  @metrics = options[:metrics]
14
- @metric_prefix = options[:metric_prefix]
15
- @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
14
+ @connpool_type = options[:connpool_type]
15
+ @connpool_type = 'connectionpool' if @connpool_type.nil? || @connpool_type == ''
16
+ @connpool_provider = options[:connpool_provider]
17
+ @connpool_provider = 'unknown' if @connpool_provider.nil? || @connpool_provider == ''
16
18
  end
17
19
 
18
20
  def with_metrics(options = {})
@@ -20,15 +22,15 @@ module Vmpooler
20
22
  start = Time.now
21
23
  conn = checkout(options)
22
24
  timespan_ms = ((Time.now - start) * 1000).to_i
23
- @metrics&.gauge(@metric_prefix + '.available', @available.length)
24
- @metrics&.timing(@metric_prefix + '.waited', timespan_ms)
25
+ @metrics&.gauge("connection_available.#{@connpool_type}.#{@connpool_provider}", @available.length)
26
+ @metrics&.timing("connection_waited.#{@connpool_type}.#{@connpool_provider}", timespan_ms)
25
27
  begin
26
28
  Thread.handle_interrupt(Exception => :immediate) do
27
29
  yield conn
28
30
  end
29
31
  ensure
30
32
  checkin
31
- @metrics&.gauge(@metric_prefix + '.available', @available.length)
33
+ @metrics&.gauge("connection_available.#{@connpool_type}.#{@connpool_provider}", @available.length)
32
34
  end
33
35
  end
34
36
  end