vm-client 1.0.0

data/lib/prometheus/client/push.rb

@@ -0,0 +1,226 @@
+# encoding: UTF-8
+
+require 'base64'
+require 'thread'
+require 'net/http'
+require 'uri'
+require 'erb'
+require 'set'
+# require 'pry'
+
+require 'prometheus/client'
+require 'prometheus/client/formats/text'
+require 'prometheus/client/label_set_validator'
+
+module Prometheus
+  # Client is a ruby implementation for a Prometheus compatible client.
+  module Client
+    # Push implements a simple way to transmit a given registry to a given
+    # Pushgateway.
+    class Push
+      class HttpError < StandardError; end
+      class HttpRedirectError < HttpError; end
+      class HttpClientError < HttpError; end
+      class HttpServerError < HttpError; end
+
+      DEFAULT_GATEWAY = 'http://localhost:9091'.freeze
+      PATH = '/metrics'.freeze
+      SUPPORTED_SCHEMES = %w(http https).freeze
+
+      attr_reader :job, :gateway, :path
+
+      def initialize(job:, gateway: DEFAULT_GATEWAY, grouping_key: {}, **kwargs)
+        raise ArgumentError, "job cannot be nil" if job.nil?
+        raise ArgumentError, "job cannot be empty" if job.empty?
+        @validator = LabelSetValidator.new(expected_labels: grouping_key.keys)
+        @validator.validate_symbols!(grouping_key)
+
+        @mutex = Mutex.new
+        @job = job
+        @gateway = gateway || DEFAULT_GATEWAY
+        @grouping_key = grouping_key
+        @path = build_path(job, grouping_key)
+
+        @uri = parse("#{@gateway}#{@path}")
+        validate_no_basic_auth!(@uri)
+
+        @http = Net::HTTP.new(@uri.host, @uri.port)
+        @http.use_ssl = (@uri.scheme == 'https')
+        @http.open_timeout = kwargs[:open_timeout] if kwargs[:open_timeout]
+        @http.read_timeout = kwargs[:read_timeout] if kwargs[:read_timeout]
+      end
+
+      def basic_auth(user, password)
+        @user = user
+        @password = password
+      end
+
+      def add(registry)
+        synchronize do
+          request(Net::HTTP::Post, registry)
+        end
+      end
+
+      def replace(registry)
+        synchronize do
+          request(Net::HTTP::Put, registry)
+        end
+      end
+
+      def delete
+        synchronize do
+          request(Net::HTTP::Delete)
+        end
+      end
+
+      private
+
+      def parse(url)
+        uri = URI.parse(url)
+
+        unless SUPPORTED_SCHEMES.include?(uri.scheme)
+          raise ArgumentError, 'only HTTP gateway URLs are supported currently.'
+        end
+
+        uri
+      rescue URI::InvalidURIError => e
+        raise ArgumentError, "#{url} is not a valid URL: #{e}"
+      end
+
+      def build_path(job, grouping_key)
+        job = job.to_s
+
+        # Job can't be empty, but it can contain `/`, so we need to base64
+        # encode it in that case
+        if job.include?('/')
+          encoded_job = Base64.urlsafe_encode64(job)
+          path = "#{PATH}/job@base64/#{encoded_job}"
+        else
+          path = "#{PATH}/job/#{ERB::Util::url_encode(job)}"
+        end
+
+        grouping_key.each do |label, value|
+          value = value.to_s
+
+          if value.include?('/')
+            encoded_value = Base64.urlsafe_encode64(value)
+            path += "/#{label}@base64/#{encoded_value}"
+          # While it's valid for the urlsafe_encode64 function to return an
+          # empty string when the input string is empty, it doesn't work for
+          # our specific use case as we're putting the result into a URL path
+          # segment. A double slash (`//`) can be normalised away by HTTP
+          # libraries, proxies, and web servers.
+          #
+          # For empty strings, we use a single padding character (`=`) as the
+          # value.
+          #
+          # See the pushgateway docs for more details:
+          #
+          # https://github.com/prometheus/pushgateway/blob/6393a901f56d4dda62cd0f6ab1f1f07c495b6354/README.md#url
+          elsif value.empty?
+            path += "/#{label}@base64/="
+          else
+            path += "/#{label}/#{ERB::Util::url_encode(value)}"
+          end
+        end
+
+        path
+      end
+
+      def request(req_class, registry = nil)
+        validate_no_label_clashes!(registry) if registry
+
+        req = req_class.new(@uri)
+        req.content_type = Formats::Text::CONTENT_TYPE
+        req.basic_auth(@user, @password) if @user
+        # binding.pry
+        compress_enabled = false
+        if registry
+          if compress_enabled
+            req['Content-Encoding'] = 'gzip'
+            gzip = Zlib::GzipWriter.new(StringIO.new)
+            gzip << Formats::Text.marshal(registry)
+            req.body = gzip.close.string
+          else
+            req.body = Formats::Text.marshal(registry)
+          end
+        end
+
+        response = @http.request(req)
+        validate_response!(response)
+
+        response
+      end
+
+      def synchronize
+        @mutex.synchronize { yield }
+      end
+
+      def validate_no_basic_auth!(uri)
+        if uri.user || uri.password
+          raise ArgumentError, <<~EOF
+            Setting Basic Auth credentials in the gateway URL is not supported, please call the `basic_auth` method.
+
+            Received username `#{uri.user}` in gateway URL. Instead of passing
+            Basic Auth credentials like this:
+
+            ```
+            push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://user:password@localhost:9091")
+            ```
+
+            please pass them like this:
+
+            ```
+            push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://localhost:9091")
+            push.basic_auth("user", "password")
+            ```
+
+            While URLs do support passing Basic Auth credentials using the
+            `http://user:password@example.com/` syntax, the username and
+            password in that syntax have to follow the usual rules for URL
+            encoding of characters per RFC 3986
+            (https://datatracker.ietf.org/doc/html/rfc3986#section-2.1).
+
+            Rather than place the burden of correctly performing that encoding
+            on users of this gem, we decided to have a separate method for
+            supplying Basic Auth credentials, with no requirement to URL encode
+            the characters in them.
+          EOF
+        end
+      end
+
+      def validate_no_label_clashes!(registry)
+        # There's nothing to check if we don't have a grouping key
+        return if @grouping_key.empty?
+
+        # We could be doing a lot of comparisons, so let's do them against a
+        # set rather than an array
+        grouping_key_labels = @grouping_key.keys.to_set
+
+        registry.metrics.each do |metric|
+          metric.labels.each do |label|
+            if grouping_key_labels.include?(label)
+              raise LabelSetValidator::InvalidLabelSetError,
+                "label :#{label} from grouping key collides with label of the " \
+                "same name from metric :#{metric.name} and would overwrite it"
+            end
+          end
+        end
+      end
+
+      def validate_response!(response)
+        status = Integer(response.code)
+        if status >= 300
+          message = "status: #{response.code}, message: #{response.message}, body: #{response.body}"
+          if status <= 399
+            raise HttpRedirectError, message
+          elsif status <= 499
+            raise HttpClientError, message
+          else
+            raise HttpServerError, message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/registry.rb

@@ -0,0 +1,100 @@
+# encoding: UTF-8
+
+require 'thread'
+
+require 'prometheus/client/counter'
+require 'prometheus/client/summary'
+require 'prometheus/client/gauge'
+require 'prometheus/client/histogram'
+require 'prometheus/client/vm_histogram'
+
+module Prometheus
+  module Client
+    # Registry
+    class Registry
+      class AlreadyRegisteredError < StandardError; end
+
+      def initialize
+        @metrics = {}
+        @mutex = Mutex.new
+      end
+
+      def register(metric)
+        name = metric.name
+
+        @mutex.synchronize do
+          if @metrics.key?(name.to_sym)
+            raise AlreadyRegisteredError, "#{name} has already been registered"
+          end
+          @metrics[name.to_sym] = metric
+        end
+
+        metric
+      end
+
+      def unregister(name)
+        @mutex.synchronize do
+          @metrics.delete(name.to_sym)
+        end
+      end
+
+      def counter(name, docstring:, labels: [], preset_labels: {}, store_settings: {})
+        register(Counter.new(name,
+                             docstring: docstring,
+                             labels: labels,
+                             preset_labels: preset_labels,
+                             store_settings: store_settings))
+      end
+
+      def summary(name, docstring:, labels: [], preset_labels: {}, store_settings: {})
+        register(Summary.new(name,
+                             docstring: docstring,
+                             labels: labels,
+                             preset_labels: preset_labels,
+                             store_settings: store_settings))
+      end
+
+      def gauge(name, docstring:, labels: [], preset_labels: {}, store_settings: {})
+        register(Gauge.new(name,
+                           docstring: docstring,
+                           labels: labels,
+                           preset_labels: preset_labels,
+                           store_settings: store_settings))
+      end
+
+      def histogram(name, docstring:, labels: [], preset_labels: {},
+                    buckets: Histogram::DEFAULT_BUCKETS,
+                    store_settings: {})
+        register(Histogram.new(name,
+                               docstring: docstring,
+                               labels: labels,
+                               preset_labels: preset_labels,
+                               buckets: buckets,
+                               store_settings: store_settings))
+      end
+
+      def vm_histogram(name, docstring:, labels: [], preset_labels: {},
+                    buckets: [],
+                    store_settings: {})
+        register(VmHistogram.new(name,
+                              docstring: docstring,
+                              labels: labels,
+                              preset_labels: preset_labels,
+                              buckets: buckets,
+                              store_settings: store_settings))
+      end
+
+      def exist?(name)
+        @mutex.synchronize { @metrics.key?(name) }
+      end
+
+      def get(name)
+        @mutex.synchronize { @metrics[name.to_sym] }
+      end
+
+      def metrics
+        @mutex.synchronize { @metrics.values }
+      end
+    end
+  end
+end

data/lib/prometheus/client/summary.rb

@@ -0,0 +1,69 @@
+# encoding: UTF-8
+
+require 'prometheus/client/metric'
+
+module Prometheus
+  module Client
+    # Summary is an accumulator for samples. It captures Numeric data and
+    # provides the total count and sum of observations.
+    class Summary < Metric
+      def type
+        :summary
+      end
+
+      # Records a given value. The recorded value is usually positive
+      # or zero. A negative value is accepted but prevents current
+      # versions of Prometheus from properly detecting counter resets
+      # in the sum of observations. See
+      # https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations
+      # for details.
+      def observe(value, labels: {})
+        base_label_set = label_set_for(labels)
+
+        @store.synchronize do
+          @store.increment(labels: base_label_set.merge(quantile: "count"), by: 1)
+          @store.increment(labels: base_label_set.merge(quantile: "sum"), by: value)
+        end
+      end
+
+      # Returns a hash with "sum" and "count" as keys
+      def get(labels: {})
+        base_label_set = label_set_for(labels)
+
+        internal_counters = ["count", "sum"]
+
+        @store.synchronize do
+          internal_counters.each_with_object({}) do |counter, acc|
+            acc[counter] = @store.get(labels: base_label_set.merge(quantile: counter))
+          end
+        end
+      end
+
+      # Returns all label sets with their values expressed as hashes with their sum/count
+      def values
+        values = @store.all_values
+
+        values.each_with_object({}) do |(label_set, v), acc|
+          actual_label_set = label_set.reject{|l| l == :quantile }
+          acc[actual_label_set] ||= { "count" => 0.0, "sum" => 0.0 }
+          acc[actual_label_set][label_set[:quantile]] = v
+        end
+      end
+
+      def init_label_set(labels)
+        base_label_set = label_set_for(labels)
+
+        @store.synchronize do
+          @store.set(labels: base_label_set.merge(quantile: "count"), val: 0)
+          @store.set(labels: base_label_set.merge(quantile: "sum"), val: 0)
+        end
+      end
+
+      private
+
+      def reserved_labels
+        [:quantile]
+      end
+    end
+  end
+end

data/lib/prometheus/client/version.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+
+module Prometheus
+  module Client
+    VERSION = '1.0.0'
+  end
+end

data/lib/prometheus/client/vm_histogram.rb

@@ -0,0 +1,164 @@
+# encoding: UTF-8
+
+require 'prometheus/client/metric'
+
+module Prometheus
+  module Client
+    # A histogram samples observations (usually things like request durations
+    # or response sizes) and counts them in dynamic VictoriaMetrics buckets. It also
+    # provides a total count and sum of all observed values.
+    class VmHistogram < Metric
+      attr_reader :buckets
+
+      E10MIN = -9
+      E10MAX = 18
+      BUCKETS_PER_DECIMAL = 18
+      DECIMAL_BUCKETS_COUNT = E10MAX - E10MIN
+      BUCKETS_COUNT = DECIMAL_BUCKETS_COUNT * BUCKETS_PER_DECIMAL
+      BUCKETS_MULTIPLIER = 10**(1.0 / BUCKETS_PER_DECIMAL)
+      VMRANGES = begin
+        h = {}
+        value = 10**E10MIN
+        range_start = format('%.3e', value)
+
+        BUCKETS_COUNT.times do |i|
+          value *= BUCKETS_MULTIPLIER
+          range_end = format('%.3e', value)
+          h[i] = "#{range_start}...#{range_end}"
+          range_start = range_end
+        end
+
+        # edge case fo zeros
+        h[-1] = '0...1.000e-09'
+
+        h
+      end
+      MAX_VMRANGE_BUCKET = VMRANGES.keys.max
+
+      def initialize(name,
+                     docstring:,
+                     labels: [],
+                     preset_labels: {},
+                     # VM histogram ignores passed buckets, accepts only for compatibility
+                     buckets: [], # rubocop:disable Lint/UnusedMethodArgument
+                     store_settings: {})
+
+        @buckets = %w[sum count]
+        # TODO: this should take into account labels
+        @non_nil_buckets = {}
+        @base_label_set_cache = {}
+
+        super(name,
+              docstring: docstring,
+              labels: labels,
+              preset_labels: preset_labels,
+              store_settings: store_settings)
+      end
+
+      def with_labels(labels)
+        new_metric = self.class.new(name,
+                                    docstring: docstring,
+                                    labels: @labels,
+                                    preset_labels: preset_labels.merge(labels),
+                                    buckets: @buckets,
+                                    store_settings: @store_settings)
+
+        # The new metric needs to use the same store as the "main" declared one, otherwise
+        # any observations on that copy with the pre-set labels won't actually be exported.
+        new_metric.replace_internal_store(@store)
+
+        new_metric
+      end
+
+      def type
+        :vm_histogram
+      end
+
+      # Records a given value. The recorded value is usually positive
+      # or zero. A negative value is ignored.
+      def observe(value, labels: {})
+        return if value.to_f.nan? || value.negative?
+
+        float_bucket_id = (Math.log10(value) - E10MIN) * BUCKETS_PER_DECIMAL
+
+        bucket_id = if float_bucket_id.negative?
+                      -1
+                    elsif float_bucket_id > MAX_VMRANGE_BUCKET
+                      MAX_VMRANGE_BUCKET
+                    else
+                      float_bucket_id.to_i
+                    end
+
+        # Edge case for 10^n values, which must go to the lower bucket
+        # according to Prometheus logic for `le`-based histograms
+        bucket_id -= 1 if (float_bucket_id - bucket_id.to_f).abs < Float::EPSILON && bucket_id.positive?
+
+        base_label_set = label_set_for(labels)
+
+        # OPTIMIZE: probably we also can use cache for vmranges to avoid using .dup every time
+        bucket_label_set = base_label_set.dup
+        bucket_label_set[:vmrange] = VMRANGES[bucket_id]
+
+        @non_nil_buckets[bucket_label_set[:vmrange]] = nil # just to track non empty buckets
+
+        unless @base_label_set_cache.key? base_label_set
+          @base_label_set_cache[base_label_set] = {
+            sum: base_label_set.merge({ le: 'sum' }),
+            count: base_label_set.merge({ le: 'count' })
+          }
+        end
+
+        @store.synchronize do
+          @store.increment(labels: bucket_label_set, by: 1)
+          @store.increment(labels: @base_label_set_cache[base_label_set][:sum], by: value)
+          @store.increment(labels: @base_label_set_cache[base_label_set][:count], by: 1)
+        end
+      end
+
+      def get(labels: {})
+        base_label_set = label_set_for(labels)
+
+        all_buckets = @non_nil_buckets.keys + buckets
+
+        @store.synchronize do
+          all_buckets.each_with_object({}) do |bucket, acc|
+            if @non_nil_buckets.key? bucket
+              value = @store.get(labels: base_label_set.merge(vmrange: bucket.to_s))
+              acc[bucket.to_s] = value if value.positive?
+            else
+              acc[bucket.to_s] = @store.get(labels: base_label_set.merge(le: bucket.to_s))
+            end
+          end
+        end
+      end
+
+      # Returns all label sets with their values expressed as hashes with their buckets
+      def values
+        values = @store.all_values
+
+        values.each_with_object({}) do |(label_set, v), acc|
+          actual_label_set = label_set.reject { |l| [:vmrange, :le].include? l }
+          acc[actual_label_set] ||= {}
+          label_name = label_set[:vmrange] || label_set[:le]
+          acc[actual_label_set][label_name.to_s] = v
+        end
+      end
+
+      def init_label_set(labels)
+        base_label_set = label_set_for(labels)
+
+        @store.synchronize do
+          @buckets.each do |bucket|
+            @store.set(labels: base_label_set.merge(le: bucket.to_s), val: 0)
+          end
+        end
+      end
+
+      private
+
+      def reserved_labels
+        [:vmrange]
+      end
+    end
+  end
+end

data/lib/prometheus/client.rb

@@ -0,0 +1,18 @@
+# encoding: UTF-8
+
+require 'prometheus/client/registry'
+require 'prometheus/client/config'
+
+module Prometheus
+  # Client is a ruby implementation for a Prometheus compatible client.
+  module Client
+    # Returns a default registry object
+    def self.registry
+      @registry ||= Registry.new
+    end
+
+    def self.config
+      @config ||= Config.new
+    end
+  end
+end

data/lib/prometheus/middleware/collector.rb

@@ -0,0 +1,103 @@
+# encoding: UTF-8
+
+require 'benchmark'
+require 'prometheus/client'
+
+module Prometheus
+  module Middleware
+    # Collector is a Rack middleware that provides a sample implementation of a
+    # HTTP tracer.
+    #
+    # By default metrics are registered on the global registry. Set the
+    # `:registry` option to use a custom registry.
+    #
+    # By default metrics all have the prefix "http_server". Set
+    # `:metrics_prefix` to something else if you like.
+    #
+    # The request counter metric is broken down by code, method and path.
+    # The request duration metric is broken down by method and path.
+    class Collector
+      attr_reader :app, :registry
+
+      def initialize(app, options = {})
+        @app = app
+        @registry = options[:registry] || Client.registry
+        @metrics_prefix = options[:metrics_prefix] || 'http_server'
+
+        init_request_metrics
+        init_exception_metrics
+      end
+
+      def call(env) # :nodoc:
+        trace(env) { @app.call(env) }
+      end
+
+      protected
+
+      def init_request_metrics
+        @requests = @registry.counter(
+          :"#{@metrics_prefix}_requests_total",
+          docstring:
+            'The total number of HTTP requests handled by the Rack application.',
+          labels: %i[code method path]
+        )
+        @durations = @registry.histogram(
+          :"#{@metrics_prefix}_request_duration_seconds",
+          docstring: 'The HTTP response duration of the Rack application.',
+          labels: %i[method path]
+        )
+      end
+
+      def init_exception_metrics
+        @exceptions = @registry.counter(
+          :"#{@metrics_prefix}_exceptions_total",
+          docstring: 'The total number of exceptions raised by the Rack application.',
+          labels: [:exception]
+        )
+      end
+
+      def trace(env)
+        response = nil
+        duration = Benchmark.realtime { response = yield }
+        record(env, response.first.to_s, duration)
+        return response
+      rescue => exception
+        @exceptions.increment(labels: { exception: exception.class.name })
+        raise
+      end
+
+      def record(env, code, duration)
+        path = generate_path(env)
+
+        counter_labels = {
+          code:   code,
+          method: env['REQUEST_METHOD'].downcase,
+          path:   path,
+        }
+
+        duration_labels = {
+          method: env['REQUEST_METHOD'].downcase,
+          path:   path,
+        }
+
+        @requests.increment(labels: counter_labels)
+        @durations.observe(duration, labels: duration_labels)
+      rescue
+        # TODO: log unexpected exception during request recording
+        nil
+      end
+
+      def generate_path(env)
+        full_path = [env['SCRIPT_NAME'], env['PATH_INFO']].join
+
+        strip_ids_from_path(full_path)
+      end
+
+      def strip_ids_from_path(path)
+        path
+          .gsub(%r{/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?=/|$)}, '/:uuid\\1')
+          .gsub(%r{/\d+(?=/|$)}, '/:id\\1')
+      end
+    end
+  end
+end