vinted-prometheus-client-mmap 1.5.0-x86_64-linux

data/lib/prometheus/client/mmaped_dict.rb

@@ -0,0 +1,83 @@
+require 'prometheus/client/helper/mmaped_file'
+require 'prometheus/client/helper/plain_file'
+require 'prometheus/client'
+
+module Prometheus
+  module Client
+    class ParsingError < StandardError
+    end
+
+    # A dict of doubles, backed by an mmapped file.
+    #
+    # The file starts with a 4 byte int, indicating how much of it is used.
+    # Then 4 bytes of padding.
+    # There's then a number of entries, consisting of a 4 byte int which is the
+    # size of the next field, a utf-8 encoded string key, padding to an 8 byte
+    # alignment, and then a 8 byte float which is the value.
+    class MmapedDict
+      attr_reader :m, :used, :positions
+
+      def self.read_all_values(f)
+        Helper::PlainFile.new(f).entries.map do |data, encoded_len, value_offset, _|
+          encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset))
+          [encoded, value]
+        end
+      end
+
+      def initialize(m)
+        @mutex = Mutex.new
+
+        @m = m
+        # @m.mlock # TODO: Ensure memory is locked to RAM
+
+        @positions = {}
+        read_all_positions.each do |key, pos|
+          @positions[key] = pos
+        end
+      rescue StandardError => e
+        raise ParsingError, "exception #{e} while processing metrics file #{path}"
+      end
+
+      def read_value(key)
+        @m.fetch_entry(@positions, key, 0.0)
+      end
+
+      def write_value(key, value)
+        @m.upsert_entry(@positions, key, value)
+      end
+
+      def write_exemplar(key, value, exemplar_id, exemplar_val)
+        @m.upsert_exemplar(@positions, key, value, exemplar_id, exemplar_val)
+      end
+
+      def path
+        @m.filepath if @m
+      end
+
+      def close
+        @m.sync
+        @m.close
+      rescue TypeError => e
+        Prometheus::Client.logger.warn("munmap raised error #{e}")
+      end
+
+      def inspect
+        "#<#{self.class}:0x#{(object_id << 1).to_s(16)}>"
+      end
+
+      private
+
+      def init_value(key)
+        @m.add_entry(@positions, key, 0.0)
+      end
+
+      # Yield (key, pos). No locking is performed.
+      def read_all_positions
+        @m.entries.map do |data, encoded_len, _, absolute_pos|
+          encoded, = data.unpack(format('@4A%d', encoded_len))
+          [encoded, absolute_pos]
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/mmaped_value.rb

@@ -0,0 +1,164 @@
+require 'prometheus/client'
+require 'prometheus/client/mmaped_dict'
+require 'json'
+
+module Prometheus
+  module Client
+    # A float protected by a mutex backed by a per-process mmaped file.
+    class MmapedValue
+      VALUE_LOCK = Mutex.new
+
+      @@files = {}
+      @@pid = -1
+
+      def initialize(type, metric_name, name, labels, multiprocess_mode = '')
+        @file_prefix = type.to_s
+        @metric_name = metric_name
+        @name = name
+        @labels = labels
+        if type == :gauge
+          @file_prefix += '_' + multiprocess_mode.to_s
+        end
+
+        @pid = -1
+
+        @mutex = Mutex.new
+        initialize_file
+      end
+
+      def increment(amount = 1, exemplar_name = '', exemplar_value = '')
+        @mutex.synchronize do
+          initialize_file if pid_changed?
+
+          @value += amount
+          write_value(@key, @value, exemplar_name, exemplar_value)
+          @value
+        end
+      end
+
+      def decrement(amount = 1)
+        increment(-amount)
+      end
+
+      def set(value)
+        @mutex.synchronize do
+          initialize_file if pid_changed?
+
+          @value = value
+          write_value(@key, @value)
+          @value
+        end
+      end
+
+      def get
+        @mutex.synchronize do
+          initialize_file if pid_changed?
+          return @value
+        end
+      end
+
+      def pid_changed?
+        @pid != Process.pid
+      end
+
+      # method needs to be run in VALUE_LOCK mutex
+      def unsafe_reinitialize_file(check_pid = true)
+        unsafe_initialize_file if !check_pid || pid_changed?
+      end
+
+      def self.reset_and_reinitialize
+        VALUE_LOCK.synchronize do
+          @@pid = Process.pid
+          @@files = {}
+
+          ObjectSpace.each_object(MmapedValue).each do |v|
+            v.unsafe_reinitialize_file(false)
+          end
+        end
+      end
+
+      def self.reset_on_pid_change
+        if pid_changed?
+          @@pid = Process.pid
+          @@files = {}
+        end
+      end
+
+      def self.reinitialize_on_pid_change
+        VALUE_LOCK.synchronize do
+          reset_on_pid_change
+
+          ObjectSpace.each_object(MmapedValue, &:unsafe_reinitialize_file)
+        end
+      end
+
+      def self.pid_changed?
+        @@pid != Process.pid
+      end
+
+      def self.multiprocess
+        true
+      end
+
+      private
+
+      def initialize_file
+        VALUE_LOCK.synchronize do
+          unsafe_initialize_file
+        end
+      end
+
+      def unsafe_initialize_file
+        self.class.reset_on_pid_change
+
+        @pid = Process.pid
+        unless @@files.has_key?(@file_prefix)
+          unless @file.nil?
+            @file.close
+          end
+          unless @exemplar_file.nil?
+            @exemplar_file.close
+          end
+          mmaped_file = Helper::MmapedFile.open_exclusive_file(@file_prefix)
+          exemplar_file = Helper::MmapedFile.open_exclusive_file('exemplar')
+
+          @@files[@file_prefix] = MmapedDict.new(mmaped_file)
+          @@files['exemplar'] = MmapedDict.new(exemplar_file)
+        end
+
+        @file = @@files[@file_prefix]
+        @exemplar_file = @@files['exemplar']
+        @key = rebuild_key
+
+        @value = read_value(@key)
+      end
+
+
+      def rebuild_key
+        keys = @labels.keys.sort
+        values = @labels.values_at(*keys)
+
+        [@metric_name, @name, keys, values].to_json
+      end
+
+      def write_value(key, val, exemplar_name = '', exemplar_value = '')
+        @file.write_value(key, val)
+        # Exemplars are only defined on counters or histograms.
+        if @file_prefix == 'counter' or @file_prefix == 'histogram' and exemplar_name != '' and exemplar_value != ''
+          @exemplar_file.write_exemplar(key, val, exemplar_name, exemplar_value)
+        end
+      rescue StandardError => e
+        Prometheus::Client.logger.warn("writing value to #{@file.path} failed with #{e}")
+        Prometheus::Client.logger.debug(e.backtrace.join("\n"))
+      end
+
+      def read_value(key)
+        @file.read_value(key)
+      rescue StandardError => e
+        Prometheus::Client.logger.warn("reading value from #{@file.path} failed with #{e}")
+        Prometheus::Client.logger.debug(e.backtrace.join("\n"))
+        0
+      end
+    end
+  end
+end

data/lib/prometheus/client/page_size.rb

@@ -0,0 +1,17 @@
+require 'open3'
+
+module Prometheus
+  module Client
+    module PageSize
+      def self.page_size(fallback_page_size: 4096)
+        stdout, status = Open3.capture2('getconf PAGESIZE')
+        return fallback_page_size if status.nil? || !status.success?
+
+        page_size = stdout.chomp.to_i
+        return fallback_page_size if page_size <= 0
+
+        page_size
+      end
+    end
+  end
+end

data/lib/prometheus/client/push.rb

@@ -0,0 +1,203 @@
+# encoding: UTF-8
+
+require 'base64'
+require 'thread'
+require 'net/http'
+require 'uri'
+require 'erb'
+require 'set'
+
+require 'prometheus/client'
+require 'prometheus/client/formats/text'
+require 'prometheus/client/label_set_validator'
+
+module Prometheus
+  # Client is a ruby implementation for a Prometheus compatible client.
+  module Client
+    # Push implements a simple way to transmit a given registry to a given
+    # Pushgateway.
+    class Push
+      class HttpError < StandardError; end
+      class HttpRedirectError < HttpError; end
+      class HttpClientError < HttpError; end
+      class HttpServerError < HttpError; end
+
+      DEFAULT_GATEWAY = 'http://localhost:9091'.freeze
+      PATH            = '/metrics/job/%s'.freeze
+      SUPPORTED_SCHEMES = %w(http https).freeze
+
+      attr_reader :job, :gateway, :path
+
+      def initialize(job:, gateway: DEFAULT_GATEWAY, grouping_key: {}, **kwargs)
+        raise ArgumentError, "job cannot be nil" if job.nil?
+        raise ArgumentError, "job cannot be empty" if job.empty?
+        @validator = LabelSetValidator.new()
+        @validator.validate(grouping_key)
+
+        @mutex = Mutex.new
+        @job = job
+        @gateway = gateway || DEFAULT_GATEWAY
+        @grouping_key = grouping_key
+        @path = build_path(job, grouping_key)
+
+        @uri = parse("#{@gateway}#{@path}")
+        validate_no_basic_auth!(@uri)
+
+        @http = Net::HTTP.new(@uri.host, @uri.port)
+        @http.use_ssl = (@uri.scheme == 'https')
+        @http.open_timeout = kwargs[:open_timeout] if kwargs[:open_timeout]
+        @http.read_timeout = kwargs[:read_timeout] if kwargs[:read_timeout]
+      end
+
+      def basic_auth(user, password)
+        @user = user
+        @password = password
+      end
+
+      def add(registry)
+        synchronize do
+          request(Net::HTTP::Post, registry)
+        end
+      end
+
+      def replace(registry)
+        synchronize do
+          request(Net::HTTP::Put, registry)
+        end
+      end
+
+      def delete
+        synchronize do
+          request(Net::HTTP::Delete)
+        end
+      end
+
+      private
+
+      def parse(url)
+        uri = URI.parse(url)
+
+        unless SUPPORTED_SCHEMES.include?(uri.scheme)
+          raise ArgumentError, 'only HTTP gateway URLs are supported currently.'
+        end
+
+        uri
+      rescue URI::InvalidURIError => e
+        raise ArgumentError, "#{url} is not a valid URL: #{e}"
+      end
+
+      def build_path(job, grouping_key)
+        path = format(PATH, ERB::Util::url_encode(job))
+
+        grouping_key.each do |label, value|
+          if value.include?('/')
+            encoded_value = Base64.urlsafe_encode64(value)
+            path += "/#{label}@base64/#{encoded_value}"
+          # While it's valid for the urlsafe_encode64 function to return an
+          # empty string when the input string is empty, it doesn't work for
+          # our specific use case as we're putting the result into a URL path
+          # segment. A double slash (`//`) can be normalised away by HTTP
+          # libraries, proxies, and web servers.
+          #
+          # For empty strings, we use a single padding character (`=`) as the
+          # value.
+          #
+          # See the pushgateway docs for more details:
+          #
+          # https://github.com/prometheus/pushgateway/blob/6393a901f56d4dda62cd0f6ab1f1f07c495b6354/README.md#url
+          elsif value.empty?
+            path += "/#{label}@base64/="
+          else
+            path += "/#{label}/#{ERB::Util::url_encode(value)}"
+          end
+        end
+
+        path
+      end
+
+      def request(req_class, registry = nil)
+        validate_no_label_clashes!(registry) if registry
+
+        req = req_class.new(@uri)
+        req.content_type = Formats::Text::CONTENT_TYPE
+        req.basic_auth(@user, @password) if @user
+        req.body = Formats::Text.marshal(registry) if registry
+
+        response = @http.request(req)
+        validate_response!(response)
+
+        response
+      end
+
+      def synchronize
+        @mutex.synchronize { yield }
+      end
+
+      def validate_no_basic_auth!(uri)
+        if uri.user || uri.password
+          raise ArgumentError, <<~EOF
+            Setting Basic Auth credentials in the gateway URL is not supported, please call the `basic_auth` method.
+
+            Received username `#{uri.user}` in gateway URL. Instead of passing
+            Basic Auth credentials like this:
+
+            ```
+            push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://user:password@localhost:9091")
+            ```
+
+            please pass them like this:
+
+            ```
+            push = Prometheus::Client::Push.new(job: "my-job", gateway: "http://localhost:9091")
+            push.basic_auth("user", "password")
+            ```
+
+            While URLs do support passing Basic Auth credentials using the
+            `http://user:password@example.com/` syntax, the username and
+            password in that syntax have to follow the usual rules for URL
+            encoding of characters per RFC 3986
+            (https://datatracker.ietf.org/doc/html/rfc3986#section-2.1).
+
+            Rather than place the burden of correctly performing that encoding
+            on users of this gem, we decided to have a separate method for
+            supplying Basic Auth credentials, with no requirement to URL encode
+            the characters in them.
+          EOF
+        end
+      end
+
+      def validate_no_label_clashes!(registry)
+        # There's nothing to check if we don't have a grouping key
+        return if @grouping_key.empty?
+
+        # We could be doing a lot of comparisons, so let's do them against a
+        # set rather than an array
+        grouping_key_labels = @grouping_key.keys.to_set
+
+        registry.metrics.each do |metric|
+            metric.values.keys.first.keys.each do |label|
+            if grouping_key_labels.include?(label)
+              raise LabelSetValidator::InvalidLabelSetError,
+                "label :#{label} from grouping key collides with label of the " \
+                "same name from metric :#{metric.name} and would overwrite it"
+            end
+          end
+        end
+      end
+
+      def validate_response!(response)
+        status = Integer(response.code)
+        if status >= 300
+          message = "status: #{response.code}, message: #{response.message}, body: #{response.body}"
+          if status <= 399
+            raise HttpRedirectError, message
+          elsif status <= 499
+            raise HttpClientError, message
+          else
+            raise HttpServerError, message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/rack/collector.rb

@@ -0,0 +1,88 @@
+# encoding: UTF-8
+
+require 'prometheus/client'
+
+module Prometheus
+  module Client
+    module Rack
+      # Collector is a Rack middleware that provides a sample implementation of
+      # a HTTP tracer. The default label builder can be modified to export a
+      # different set of labels per recorded metric.
+      class Collector
+        attr_reader :app, :registry
+
+        def initialize(app, options = {}, &label_builder)
+          @app = app
+          @registry = options[:registry] || Client.registry
+          @label_builder = label_builder || DEFAULT_LABEL_BUILDER
+
+          init_request_metrics
+          init_exception_metrics
+        end
+
+        def call(env) # :nodoc:
+          trace(env) { @app.call(env) }
+        end
+
+        protected
+
+        DEFAULT_LABEL_BUILDER = proc do |env|
+          {
+            method: env['REQUEST_METHOD'].downcase,
+            host:   env['HTTP_HOST'].to_s,
+            path:   env['PATH_INFO'].to_s,
+          }
+        end
+
+        def init_request_metrics
+          @requests = @registry.counter(
+            :http_requests_total,
+            'A counter of the total number of HTTP requests made.',
+          )
+          @durations = @registry.summary(
+            :http_request_duration_seconds,
+            'A summary of the response latency.',
+          )
+          @durations_hist = @registry.histogram(
+            :http_req_duration_seconds,
+            'A histogram of the response latency.',
+          )
+        end
+
+        def init_exception_metrics
+          @exceptions = @registry.counter(
+            :http_exceptions_total,
+            'A counter of the total number of exceptions raised.',
+          )
+        end
+
+        def trace(env)
+          start = Time.now
+          yield.tap do |response|
+            duration = (Time.now - start).to_f
+            record(labels(env, response), duration)
+          end
+        rescue => exception
+          @exceptions.increment(exception: exception.class.name)
+          raise
+        end
+
+        def labels(env, response)
+          @label_builder.call(env).tap do |labels|
+            labels[:code] = response.first.to_s
+          end
+        end
+
+        def record(labels, duration)
+          @requests.increment(labels)
+          @durations.observe(labels, duration)
+          @durations_hist.observe(labels, duration)
+        rescue => exception
+          @exceptions.increment(exception: exception.class.name)
+          raise
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/rack/exporter.rb

@@ -0,0 +1,102 @@
+# encoding: UTF-8
+
+require 'prometheus/client'
+require 'prometheus/client/formats/text'
+require 'prometheus/client/formats/protobuf'
+
+module Prometheus
+  module Client
+    module Rack
+      # Exporter is a Rack middleware that provides a sample implementation of
+      # a Prometheus HTTP client API.
+      class Exporter
+        attr_reader :app, :registry, :path
+
+        FALLBACK = Formats::Text
+
+        def initialize(app, options = {})
+          @app = app
+          @registry = options[:registry] || Client.registry
+          @path = options[:path] || '/metrics'
+
+          if Prometheus::Client.configuration.enable_protobuf
+            @formats = [Formats::Text, Formats::Protobuf]
+          else
+            @formats = [Formats::Text]
+          end
+          @acceptable = build_dictionary(@formats, FALLBACK)
+        end
+
+        def call(env)
+          if env['PATH_INFO'] == @path
+            format = negotiate(env['HTTP_ACCEPT'], @acceptable)
+            format ? respond_with(format) : not_acceptable(@formats)
+          else
+            @app.call(env)
+          end
+        end
+
+        private
+
+        def negotiate(accept, formats)
+          accept = '*/*' if accept.to_s.empty?
+
+          parse(accept).each do |content_type, _|
+            return formats[content_type] if formats.key?(content_type)
+          end
+
+          nil
+        end
+
+        def parse(header)
+          header.to_s.split(/\s*,\s*/).map do |type|
+            attributes = type.split(/\s*;\s*/)
+            quality = extract_quality(attributes)
+
+            [attributes.join('; '), quality]
+          end.sort_by(&:last).reverse
+        end
+
+        def extract_quality(attributes, default = 1.0)
+          quality = default
+
+          attributes.delete_if do |attr|
+            quality = attr.split('q=').last.to_f if attr.start_with?('q=')
+          end
+
+          quality
+        end
+
+        def respond_with(format)
+          response = if Prometheus::Client.configuration.value_class.multiprocess
+                       format.marshal_multiprocess
+                     else
+                       format.marshal
+                     end
+          [
+            200,
+            { 'Content-Type' => format::CONTENT_TYPE },
+            [response],
+          ]
+        end
+
+        def not_acceptable(formats)
+          types = formats.map { |format| format::MEDIA_TYPE }
+
+          [
+            406,
+            { 'Content-Type' => 'text/plain' },
+            ["Supported media types: #{types.join(', ')}"],
+          ]
+        end
+
+        def build_dictionary(formats, fallback)
+          formats.each_with_object('*/*' => fallback) do |format, memo|
+            memo[format::CONTENT_TYPE] = format
+            memo[format::MEDIA_TYPE] = format
+          end
+        end
+      end
+    end
+  end
+end