vmpooler 3.7.0 → 3.9.0

data/lib/vmpooler/api/v3.rb

@@ -9,6 +9,20 @@ module Vmpooler
       api_version = '3'
       api_prefix  = "/api/v#{api_version}"
 
+      # Simple in-memory cache for status endpoint
+      # rubocop:disable Style/ClassVars
+      @@status_cache = {}
+      @@status_cache_mutex = Mutex.new
+      # rubocop:enable Style/ClassVars
+      STATUS_CACHE_TTL = 30 # seconds
+
+      # Clear cache (useful for testing)
+      def self.clear_status_cache
+        @@status_cache_mutex.synchronize do
+          @@status_cache.clear
+        end
+      end
+
       helpers do
         include Vmpooler::API::Helpers
       end
@@ -464,6 +478,32 @@ module Vmpooler
         end
       end
 
+      # Cache helper methods for status endpoint
+      def get_cached_status(cache_key)
+        @@status_cache_mutex.synchronize do
+          cached = @@status_cache[cache_key]
+          if cached && (Time.now - cached[:timestamp]) < STATUS_CACHE_TTL
+            return cached[:data]
+          end
+
+          nil
+        end
+      end
+
+      def set_cached_status(cache_key, data)
+        @@status_cache_mutex.synchronize do
+          @@status_cache[cache_key] = {
+            data: data,
+            timestamp: Time.now
+          }
+          # Cleanup old cache entries (keep only last 10 unique view combinations)
+          if @@status_cache.size > 10
+            oldest = @@status_cache.min_by { |_k, v| v[:timestamp] }
+            @@status_cache.delete(oldest[0])
+          end
+        end
+      end
+
       def sync_pool_templates
         tracer.in_span("Vmpooler::API::V3.#{__method__}") do
           pool_index = pool_index(pools)
@@ -646,6 +686,13 @@ module Vmpooler
       get "#{api_prefix}/status/?" do
         content_type :json
 
+        # Create cache key based on view parameters
+        cache_key = params[:view] ? "status_#{params[:view]}" : "status_all"
+
+        # Try to get cached response
+        cached_response = get_cached_status(cache_key)
+        return cached_response if cached_response
+
         if params[:view]
           views = params[:view].split(",")
         end
@@ -706,7 +753,12 @@ module Vmpooler
 
         result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
 
-        JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
+        response = JSON.pretty_generate(Hash[result.sort_by { |k, _v| k }])
+
+        # Cache the response
+        set_cached_status(cache_key, response)
+
+        response
       end
 
       # request statistics for specific pools by passing parameter 'pool'
@@ -1085,9 +1137,29 @@ module Vmpooler
         result = { 'ok' => false }
         metrics.increment('http_requests_vm_total.post.vm.checkout')
 
-        payload = JSON.parse(request.body.read)
+        # Validate and sanitize JSON body
+        payload = sanitize_json_body(request.body.read)
+        if validation_error?(payload)
+          status 400
+          return JSON.pretty_generate(payload)
+        end
 
-        if payload
+        # Validate each template and count
+        payload.each do |template, count|
+          validation = validate_pool_name(template)
+          if validation_error?(validation)
+            status 400
+            return JSON.pretty_generate(validation)
+          end
+
+          validated_count = validate_vm_count(count)
+          if validation_error?(validated_count)
+            status 400
+            return JSON.pretty_generate(validated_count)
+          end
+        end
+
+        if payload && !payload.empty?
           invalid = invalid_templates(payload)
           if invalid.empty?
             result = atomically_allocate_vms(payload)
@@ -1206,6 +1278,7 @@ module Vmpooler
         result = { 'ok' => false }
         metrics.increment('http_requests_vm_total.get.vm.template')
 
+        # Template can contain multiple pools separated by +, so validate after parsing
         payload = extract_templates_from_query_params(params[:template])
 
         if payload
@@ -1235,6 +1308,13 @@ module Vmpooler
         status 404
         result['ok'] = false
 
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
         params[:hostname] = hostname_shorten(params[:hostname])
 
         rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}")
@@ -1373,6 +1453,13 @@ module Vmpooler
         status 404
         result['ok'] = false
 
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
         params[:hostname] = hostname_shorten(params[:hostname])
 
         rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}")
@@ -1403,16 +1490,21 @@ module Vmpooler
 
         failure = []
 
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
         params[:hostname] = hostname_shorten(params[:hostname])
 
         if backend.exists?("vmpooler__vm__#{params[:hostname]}")
-          begin
-            jdata = JSON.parse(request.body.read)
-          rescue StandardError => e
-            span = OpenTelemetry::Trace.current_span
-            span.record_exception(e)
-            span.status = OpenTelemetry::Trace::Status.error(e.to_s)
-            halt 400, JSON.pretty_generate(result)
+          # Validate and sanitize JSON body
+          jdata = sanitize_json_body(request.body.read)
+          if validation_error?(jdata)
+            status 400
+            return JSON.pretty_generate(jdata)
           end
 
           # Validate data payload
@@ -1421,6 +1513,13 @@ module Vmpooler
               when 'lifetime'
                 need_token! if Vmpooler::API.settings.config[:auth]
 
+                # Validate lifetime is a positive integer
+                lifetime_int = arg.to_i
+                if lifetime_int <= 0
+                  failure.push("Lifetime must be a positive integer (got #{arg})")
+                  next
+                end
+
                 # in hours, defaults to one week
                 max_lifetime_upper_limit = config['max_lifetime_upper_limit']
                 if max_lifetime_upper_limit
@@ -1430,13 +1529,17 @@ module Vmpooler
                   end
                 end
 
-                # validate lifetime is within boundaries
-                unless arg.to_i > 0
-                  failure.push("You provided a lifetime (#{arg}) but you must provide a positive number.")
-                end
-
               when 'tags'
                 failure.push("You provided tags (#{arg}) as something other than a hash.") unless arg.is_a?(Hash)
+
+                # Validate each tag key and value
+                arg.each do |key, value|
+                  tag_validation = validate_tag(key, value)
+                  if validation_error?(tag_validation)
+                    failure.push(tag_validation['error'])
+                  end
+                end
+
                 failure.push("You provided unsuppored tags (#{arg}).") if config['allowed_tags'] && !(arg.keys - config['allowed_tags']).empty?
               else
                 failure.push("Unknown argument #{arg}.")
@@ -1478,9 +1581,23 @@ module Vmpooler
         status 404
         result = { 'ok' => false }
 
+        # Validate hostname
+        validation = validate_hostname(params[:hostname])
+        if validation_error?(validation)
+          status 400
+          return JSON.pretty_generate(validation)
+        end
+
+        # Validate disk size
+        validated_size = validate_disk_size(params[:size])
+        if validation_error?(validated_size)
+          status 400
+          return JSON.pretty_generate(validated_size)
+        end
+
         params[:hostname] = hostname_shorten(params[:hostname])
 
-        if ((params[:size].to_i > 0 )and (backend.exists?("vmpooler__vm__#{params[:hostname]}")))
+        if backend.exists?("vmpooler__vm__#{params[:hostname]}")
           result[params[:hostname]] = {}
           result[params[:hostname]]['disk'] = "+#{params[:size]}gb"
 

data/lib/vmpooler/circuit_breaker.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Vmpooler
+  # Circuit breaker pattern implementation to prevent cascading failures
+  # when a provider becomes unresponsive or experiences repeated failures.
+  #
+  # States:
+  # - CLOSED: Normal operation, requests flow through
+  # - OPEN: Provider is failing, reject requests immediately (fail fast)
+  # - HALF_OPEN: Testing if provider has recovered with limited requests
+  class CircuitBreaker
+    STATES = %i[closed open half_open].freeze
+
+    class CircuitOpenError < StandardError; end
+
+    attr_reader :state, :failure_count, :success_count
+
+    # Initialize a new circuit breaker
+    #
+    # @param name [String] Name for logging/metrics (e.g., "vsphere_provider")
+    # @param logger [Object] Logger instance
+    # @param metrics [Object] Metrics instance
+    # @param failure_threshold [Integer] Number of failures before opening circuit
+    # @param timeout [Integer] Seconds to wait in open state before testing (half-open)
+    # @param half_open_attempts [Integer] Number of successful test requests needed to close
+    def initialize(name:, logger:, metrics:, failure_threshold: 5, timeout: 30, half_open_attempts: 3)
+      @name = name
+      @logger = logger
+      @metrics = metrics
+      @failure_threshold = failure_threshold
+      @timeout = timeout
+      @half_open_attempts = half_open_attempts
+
+      @state = :closed
+      @failure_count = 0
+      @success_count = 0
+      @last_failure_time = nil
+      @mutex = Mutex.new
+    end
+
+    # Execute a block with circuit breaker protection
+    #
+    # @yield Block to execute if circuit allows
+    # @return Result of the block
+    # @raise CircuitOpenError if circuit is open and timeout hasn't elapsed
+    def call
+      check_state
+
+      begin
+        result = yield
+        on_success
+        result
+      rescue StandardError => e
+        on_failure(e)
+        raise
+      end
+    end
+
+    # Check if circuit allows requests
+    # @return [Boolean] true if circuit is closed or half-open
+    def allow_request?
+      @mutex.synchronize do
+        case @state
+        when :closed
+          true
+        when :half_open
+          true
+        when :open
+          if should_attempt_reset?
+            true
+          else
+            false
+          end
+        end
+      end
+    end
+
+    # Get current circuit breaker status
+    # @return [Hash] Status information
+    def status
+      @mutex.synchronize do
+        {
+          name: @name,
+          state: @state,
+          failure_count: @failure_count,
+          success_count: @success_count,
+          last_failure_time: @last_failure_time,
+          next_retry_time: next_retry_time
+        }
+      end
+    end
+
+    private
+
+    def check_state
+      @mutex.synchronize do
+        case @state
+        when :open
+          if should_attempt_reset?
+            transition_to_half_open
+          else
+            time_remaining = (@timeout - (Time.now - @last_failure_time)).round(1)
+            raise CircuitOpenError, "Circuit breaker '#{@name}' is open (#{@failure_count} failures, retry in #{time_remaining}s)"
+          end
+        when :half_open
+          # Allow limited requests through for testing
+        when :closed
+          # Normal operation
+        end
+      end
+    end
+
+    def should_attempt_reset?
+      return false unless @last_failure_time
+
+      Time.now - @last_failure_time >= @timeout
+    end
+
+    def next_retry_time
+      return nil unless @last_failure_time && @state == :open
+
+      @last_failure_time + @timeout
+    end
+
+    def on_success
+      @mutex.synchronize do
+        case @state
+        when :closed
+          # Reset failure count on success in closed state
+          @failure_count = 0 if @failure_count > 0
+        when :half_open
+          @success_count += 1
+          @failure_count = 0
+          @logger.log('d', "[+] [circuit_breaker] '#{@name}' successful test request (#{@success_count}/#{@half_open_attempts})")
+
+          transition_to_closed if @success_count >= @half_open_attempts
+        when :open
+          # Should not happen, but reset if we somehow get a success
+          transition_to_closed
+        end
+      end
+    end
+
+    def on_failure(error)
+      @mutex.synchronize do
+        @failure_count += 1
+        @last_failure_time = Time.now
+
+        case @state
+        when :closed
+          @logger.log('d', "[!] [circuit_breaker] '#{@name}' failure #{@failure_count}/#{@failure_threshold}: #{error.class}")
+          transition_to_open if @failure_count >= @failure_threshold
+        when :half_open
+          @logger.log('d', "[!] [circuit_breaker] '#{@name}' failed during half-open test")
+          transition_to_open
+        when :open
+          # Already open, just log
+          @logger.log('d', "[!] [circuit_breaker] '#{@name}' additional failure while open")
+        end
+      end
+    end
+
+    def transition_to_open
+      @state = :open
+      @success_count = 0
+      @logger.log('s', "[!] [circuit_breaker] '#{@name}' OPENED after #{@failure_count} failures (will retry in #{@timeout}s)")
+      @metrics.increment("circuit_breaker.opened.#{@name}")
+      @metrics.gauge("circuit_breaker.state.#{@name}", 1) # 1 = open
+    end
+
+    def transition_to_half_open
+      @state = :half_open
+      @success_count = 0
+      @failure_count = 0
+      @logger.log('s', "[*] [circuit_breaker] '#{@name}' HALF-OPEN, testing provider health")
+      @metrics.increment("circuit_breaker.half_open.#{@name}")
+      @metrics.gauge("circuit_breaker.state.#{@name}", 0.5) # 0.5 = half-open
+    end
+
+    def transition_to_closed
+      @state = :closed
+      @failure_count = 0
+      @success_count = 0
+      @logger.log('s', "[+] [circuit_breaker] '#{@name}' CLOSED, provider recovered")
+      @metrics.increment("circuit_breaker.closed.#{@name}")
+      @metrics.gauge("circuit_breaker.state.#{@name}", 0) # 0 = closed
+    end
+  end
+end

data/lib/vmpooler/generic_connection_pool.rb

@@ -34,6 +34,34 @@ module Vmpooler
           end
         end
       end
+
+      # Get connection pool health status
+      # @return [Hash] Health status including utilization and queue depth
+      def health_status
+        {
+          size: @size,
+          available: @available.length,
+          in_use: @size - @available.length,
+          utilization: ((@size - @available.length).to_f / @size * 100).round(2),
+          waiting_threads: (@queue.respond_to?(:length) ? @queue.length : 0),
+          state: determine_health_state
+        }
+      end
+
+      private
+
+      def determine_health_state
+        utilization = ((@size - @available.length).to_f / @size * 100)
+        waiting = @queue.respond_to?(:length) ? @queue.length : 0
+
+        if utilization >= 90 || waiting > 5
+          :critical  # Pool exhausted or many waiting threads
+        elsif utilization >= 70 || waiting > 2
+          :warning   # Pool under stress
+        else
+          :healthy   # Normal operation
+        end
+      end
     end
   end
 end

data/lib/vmpooler/metrics/promstats.rb

@@ -329,6 +329,66 @@ module Vmpooler
             buckets: REDIS_CONNECT_BUCKETS,
             docstring: 'vmpooler redis connection wait time',
             param_labels: %i[type provider]
+          },
+          vmpooler_health: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler health check metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_purge: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler purge metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_destroy: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler destroy metrics',
+            param_labels: %i[poolname]
+          },
+          vmpooler_clone: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler clone metrics',
+            param_labels: %i[poolname]
+          },
+          circuit_breaker: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'Circuit breaker state and failure tracking',
+            param_labels: %i[metric_path]
+          },
+          connection_pool: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'Connection pool health metrics',
+            param_labels: %i[metric_path]
+          },
+          adaptive_timeout: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'Adaptive timeout statistics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_performance: {
+            mtype: M_GAUGE,
+            torun: %i[manager],
+            docstring: 'vmpooler performance metrics for pool operations',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_dlq: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'vmpooler dead letter queue metrics',
+            param_labels: %i[metric_path]
+          },
+          vmpooler_errors: {
+            mtype: M_COUNTER,
+            torun: %i[manager],
+            docstring: 'vmpooler error counters including permanent failures',
+            param_labels: %i[metric_path]
           }
         }
       end