vinted-prometheus-client-mmap 1.5.0-x86_64-linux

data/lib/prometheus/client/helper/entry_parser.rb

@@ -0,0 +1,132 @@
+require 'prometheus/client/helper/json_parser'
+
+module Prometheus
+  module Client
+    module Helper
+      module EntryParser
+        class ParsingError < RuntimeError;
+        end
+
+        MINIMUM_SIZE = 8
+        START_POSITION = 8
+        VALUE_BYTES = 8
+        ENCODED_LENGTH_BYTES = 4
+
+        def used
+          slice(0..3).unpack('l')[0]
+        end
+
+        def parts
+          @parts ||= File.basename(filepath, '.db')
+                       .split('_')
+                       .map { |e| e.gsub(/-\d+$/, '') } # remove trailing -number
+        end
+
+        def type
+          parts[0].to_sym
+        end
+
+        def pid
+          (parts[2..-1] || []).join('_')
+        end
+
+        def multiprocess_mode
+          parts[1]
+        end
+
+        def empty?
+          size < MINIMUM_SIZE || used.zero?
+        end
+
+        def entries(ignore_errors = false)
+          return Enumerator.new {} if empty?
+
+          Enumerator.new do |yielder|
+            used_ = used # cache used to avoid unnecessary unpack operations
+
+            pos = START_POSITION # used + padding offset
+            while pos < used_ && pos < size && pos > 0
+              data = slice(pos..-1)
+              unless data
+                raise ParsingError, "data slice is nil at pos #{pos}" unless ignore_errors
+                pos += 8
+                next
+              end
+
+              encoded_len, first_encoded_bytes = data.unpack('LL')
+              if encoded_len.nil? || encoded_len.zero? || first_encoded_bytes.nil? || first_encoded_bytes.zero?
+                # do not parse empty data
+                pos += 8
+                next
+              end
+
+              entry_len = ENCODED_LENGTH_BYTES + encoded_len
+              padding_len = 8 - entry_len % 8
+
+              value_offset = entry_len + padding_len # align to 8 bytes
+              pos += value_offset
+
+              if value_offset > 0 && (pos + VALUE_BYTES) <= size # if positions are safe
+                yielder.yield data, encoded_len, value_offset, pos
+              else
+                raise ParsingError, "data slice is nil at pos #{pos}" unless ignore_errors
+              end
+              pos += VALUE_BYTES
+            end
+          end
+        end
+
+        def parsed_entries(ignore_errors = false)
+          result = entries(ignore_errors).map do |data, encoded_len, value_offset, _|
+            begin
+              encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset))
+              [encoded, value]
+            rescue ArgumentError => e
+              Prometheus::Client.logger.debug("Error processing data: #{bin_to_hex(data[0, 7])} len: #{encoded_len} value_offset: #{value_offset}")
+              raise ParsingError, e unless ignore_errors
+            end
+          end
+
+          result.reject!(&:nil?) if ignore_errors
+          result
+        end
+
+        def to_metrics(metrics = {}, ignore_errors = false)
+          parsed_entries(ignore_errors).each do |key, value|
+            begin
+              metric_name, name, labelnames, labelvalues = JsonParser.load(key)
+              labelnames ||= []
+              labelvalues ||= []
+
+              metric = metrics.fetch(metric_name,
+                                     metric_name: metric_name,
+                                     help: 'Multiprocess metric',
+                                     type: type,
+                                     samples: [])
+              if type == :gauge
+                metric[:multiprocess_mode] = multiprocess_mode
+                metric[:samples] += [[name, labelnames.zip(labelvalues) + [['pid', pid]], value]]
+              else
+                # The duplicates and labels are fixed in the next for.
+                metric[:samples] += [[name, labelnames.zip(labelvalues), value]]
+              end
+              metrics[metric_name] = metric
+
+            rescue JSON::ParserError => e
+              raise ParsingError(e) unless ignore_errors
+            end
+          end
+
+          metrics.reject! { |e| e.nil? } if ignore_errors
+          metrics
+        end
+
+        private
+
+        def bin_to_hex(s)
+          s.each_byte.map { |b| b.to_s(16) }.join
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/file_locker.rb

@@ -0,0 +1,50 @@
+module Prometheus
+  module Client
+    module Helper
+      class FileLocker
+        class << self
+          LOCK_FILE_MUTEX = Mutex.new
+
+          def lock_to_process(filepath)
+            LOCK_FILE_MUTEX.synchronize do
+              @file_locks ||= {}
+              return false if @file_locks[filepath]
+
+              file = File.open(filepath, 'ab')
+              if file.flock(File::LOCK_NB | File::LOCK_EX)
+                @file_locks[filepath] = file
+                return true
+              else
+                return false
+              end
+            end
+          end
+
+          def unlock(filepath)
+            LOCK_FILE_MUTEX.synchronize do
+              @file_locks ||= {}
+              return false unless @file_locks[filepath]
+
+              file = @file_locks[filepath]
+              file.flock(File::LOCK_UN)
+              file.close
+              @file_locks.delete(filepath)
+            end
+          end
+
+          def unlock_all
+            LOCK_FILE_MUTEX.synchronize do
+              @file_locks ||= {}
+              @file_locks.values.each do |file|
+                file.flock(File::LOCK_UN)
+                file.close
+              end
+
+              @file_locks = {}
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/json_parser.rb

@@ -0,0 +1,23 @@
+require 'json'
+
+module Prometheus
+  module Client
+    module Helper
+      module JsonParser
+        class << self
+          if defined?(Oj)
+            def load(s)
+              Oj.load(s)
+            rescue Oj::ParseError, EncodingError => e
+              raise JSON::ParserError.new(e.message)
+            end
+          else
+            def load(s)
+              JSON.parse(s)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/metrics_processing.rb

@@ -0,0 +1,45 @@
+module Prometheus
+  module Client
+    module Helper
+      module MetricsProcessing
+        def self.merge_metrics(metrics)
+          metrics.each_value do |metric|
+            metric[:samples] = merge_samples(metric[:samples], metric[:type], metric[:multiprocess_mode]).map do |(name, labels), value|
+              [name, labels.to_h, value]
+            end
+          end
+        end
+
+        def self.merge_samples(raw_samples, metric_type, multiprocess_mode)
+          samples = {}
+          raw_samples.each do |name, labels, value|
+            without_pid = labels.reject { |l| l[0] == 'pid' }
+
+            case metric_type
+              when :gauge
+                case multiprocess_mode
+                  when 'min'
+                    s = samples.fetch([name, without_pid], value)
+                    samples[[name, without_pid]] = [s, value].min
+                  when 'max'
+                    s = samples.fetch([name, without_pid], value)
+                    samples[[name, without_pid]] = [s, value].max
+                  when 'livesum'
+                    s = samples.fetch([name, without_pid], 0.0)
+                    samples[[name, without_pid]] = s + value
+                  else # all/liveall
+                    samples[[name, labels]] = value
+                end
+              else
+                # Counter, Histogram and Summary.
+                s = samples.fetch([name, without_pid], 0.0)
+                samples[[name, without_pid]] = s + value
+            end
+          end
+
+          samples
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/metrics_representation.rb

@@ -0,0 +1,51 @@
+module Prometheus
+  module Client
+    module Helper
+      module MetricsRepresentation
+        METRIC_LINE = '%s%s %s'.freeze
+        TYPE_LINE = '# TYPE %s %s'.freeze
+        HELP_LINE = '# HELP %s %s'.freeze
+
+        LABEL = '%s="%s"'.freeze
+        SEPARATOR = ','.freeze
+        DELIMITER = "\n".freeze
+
+        REGEX = { doc: /[\n\\]/, label: /[\n\\"]/ }.freeze
+        REPLACE = { "\n" => '\n', '\\' => '\\\\', '"' => '\"' }.freeze
+
+        def self.to_text(metrics)
+          lines = []
+
+          metrics.each do |name, metric|
+            lines << format(HELP_LINE, name, escape(metric[:help]))
+            lines << format(TYPE_LINE, name, metric[:type])
+            metric[:samples].each do |metric_name, labels, value|
+              lines << metric(metric_name, format_labels(labels), value)
+            end
+          end
+
+          # there must be a trailing delimiter
+          (lines << nil).join(DELIMITER)
+        end
+
+        def self.metric(name, labels, value)
+          format(METRIC_LINE, name, labels, value)
+        end
+
+        def self.format_labels(set)
+          return if set.empty?
+
+          strings = set.each_with_object([]) do |(key, value), memo|
+            memo << format(LABEL, key, escape(value, :label))
+          end
+
+          "{#{strings.join(SEPARATOR)}}"
+        end
+
+        def self.escape(string, format = :doc)
+          string.to_s.gsub(REGEX[format], REPLACE)
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/mmaped_file.rb

@@ -0,0 +1,64 @@
+require 'prometheus/client/helper/entry_parser'
+require 'prometheus/client/helper/file_locker'
+
+# load precompiled extension if available
+begin
+  ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
+  require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
+rescue LoadError
+  require 'fast_mmaped_file_rs'
+end
+
+module Prometheus
+  module Client
+    module Helper
+      class MmapedFile < FastMmapedFileRs
+        include EntryParser
+
+        attr_reader :filepath, :size
+
+        def initialize(filepath)
+          @filepath = filepath
+
+          File.open(filepath, 'a+b') do |file|
+            file.truncate(initial_mmap_file_size) if file.size < MINIMUM_SIZE
+            @size = file.size
+          end
+
+          super(filepath)
+        end
+
+        def close
+          munmap
+          FileLocker.unlock(filepath)
+        end
+
+        private
+
+        def initial_mmap_file_size
+          Prometheus::Client.configuration.initial_mmap_file_size
+        end
+
+        public
+
+        class << self
+          def open(filepath)
+            MmapedFile.new(filepath)
+          end
+
+          def ensure_exclusive_file(file_prefix = 'mmaped_file')
+            (0..Float::INFINITY).lazy
+              .map { |f_num| "#{file_prefix}_#{Prometheus::Client.pid}-#{f_num}.db" }
+              .map { |filename| File.join(Prometheus::Client.configuration.multiprocess_files_dir, filename) }
+              .find { |path| Helper::FileLocker.lock_to_process(path) }
+          end
+
+          def open_exclusive_file(file_prefix = 'mmaped_file')
+            filename = Helper::MmapedFile.ensure_exclusive_file(file_prefix)
+            open(filename)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/plain_file.rb

@@ -0,0 +1,29 @@
+require 'prometheus/client/helper/entry_parser'
+
+module Prometheus
+  module Client
+    module Helper
+      # Parses DB files without using mmap
+      class PlainFile
+        include EntryParser
+        attr_reader :filepath
+
+        def source
+          @data ||= File.read(filepath, mode: 'rb')
+        end
+
+        def initialize(filepath)
+          @filepath = filepath
+        end
+
+        def slice(*args)
+          source.slice(*args)
+        end
+
+        def size
+          source.length
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/histogram.rb

@@ -0,0 +1,80 @@
+require 'prometheus/client/metric'
+require 'prometheus/client/uses_value_type'
+
+module Prometheus
+  module Client
+    # A histogram samples observations (usually things like request durations
+    # or response sizes) and counts them in configurable buckets. It also
+    # provides a sum of all observed values.
+    class Histogram < Metric
+      # Value represents the state of a Histogram at a given point.
+      class Value < Hash
+        include UsesValueType
+        attr_accessor :sum, :total, :total_inf
+
+        def initialize(type, name, labels, buckets)
+          @sum = value_object(type, name, "#{name}_sum", labels)
+          @total = value_object(type, name, "#{name}_count", labels)
+          @total_inf = value_object(type, name, "#{name}_bucket", labels.merge(le: "+Inf"))
+
+          buckets.each do |bucket|
+            self[bucket] = value_object(type, name, "#{name}_bucket", labels.merge(le: bucket.to_s))
+          end
+        end
+
+        def observe(value, exemplar_name = '', exemplar_value = '')
+          @sum.increment(value, exemplar_name, exemplar_value)
+          @total.increment(1, exemplar_name, exemplar_value)
+          @total_inf.increment(1, exemplar_name, exemplar_value)
+
+          each_key do |bucket|
+            self[bucket].increment(1, exemplar_name, exemplar_value) if value <= bucket
+          end
+        end
+
+        def get()
+          hash = {}
+          each_key do |bucket|
+            hash[bucket] = self[bucket].get()
+          end
+          hash
+        end
+      end
+
+      # DEFAULT_BUCKETS are the default Histogram buckets. The default buckets
+      # are tailored to broadly measure the response time (in seconds) of a
+      # network service. (From DefBuckets client_golang)
+      DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1,
+                         2.5, 5, 10].freeze
+
+      # Offer a way to manually specify buckets
+      def initialize(name, docstring, base_labels = {},
+                     buckets = DEFAULT_BUCKETS)
+        raise ArgumentError, 'Unsorted buckets, typo?' unless sorted? buckets
+
+        @buckets = buckets
+        super(name, docstring, base_labels)
+      end
+
+      def type
+        :histogram
+      end
+
+      def observe(labels, value, exemplar_name = '', exemplar_value = '')
+        label_set = label_set_for(labels)
+        synchronize { @values[label_set].observe(value, exemplar_name, exemplar_value) }
+      end
+
+      private
+
+      def default(labels)
+        # TODO: default function needs to know key of hash info (label names and values)
+        Value.new(type, @name, labels, @buckets)
+      end
+
+      def sorted?(bucket)
+        bucket.each_cons(2).all? { |i, j| i <= j }
+      end
+    end
+  end
+end

data/lib/prometheus/client/label_set_validator.rb

@@ -0,0 +1,85 @@
+# encoding: UTF-8
+
+module Prometheus
+  module Client
+    # LabelSetValidator ensures that all used label sets comply with the
+    # Prometheus specification.
+    class LabelSetValidator
+      RESERVED_LABELS = [].freeze
+
+      class LabelSetError < StandardError; end
+      class InvalidLabelSetError < LabelSetError; end
+      class InvalidLabelError < LabelSetError; end
+      class ReservedLabelError < LabelSetError; end
+
+      def initialize(reserved_labels = [])
+        @reserved_labels = (reserved_labels + RESERVED_LABELS).freeze
+        @validated = {}
+      end
+
+      def valid?(labels)
+        unless labels.is_a?(Hash)
+          raise InvalidLabelSetError, "#{labels} is not a valid label set"
+        end
+
+        labels.all? do |key, value|
+          validate_symbol(key)
+          validate_name(key)
+          validate_reserved_key(key)
+          validate_value(key, value)
+        end
+      end
+
+      def validate(labels)
+        return labels if @validated.key?(labels.hash)
+
+        valid?(labels)
+
+        unless @validated.empty? || match?(labels, @validated.first.last)
+          raise InvalidLabelSetError, "labels must have the same signature: (#{label_diff(labels, @validated.first.last)})"
+        end
+
+        @validated[labels.hash] = labels
+      end
+
+      private
+
+      def label_diff(a, b)
+        "expected keys: #{b.keys.sort}, got: #{a.keys.sort}"
+      end
+
+      def match?(a, b)
+        a.keys.sort == b.keys.sort
+      end
+
+      def validate_symbol(key)
+        return true if key.is_a?(Symbol)
+
+        raise InvalidLabelError, "label #{key} is not a symbol"
+      end
+
+      def validate_name(key)
+        return true unless key.to_s.start_with?('__')
+
+        raise ReservedLabelError, "label #{key} must not start with __"
+      end
+
+      def validate_reserved_key(key)
+        return true unless @reserved_labels.include?(key)
+
+        raise ReservedLabelError, "#{key} is reserved"
+      end
+
+      def validate_value(key, value)
+        return true if value.is_a?(String) ||
+                       value.is_a?(Numeric) ||
+                       value.is_a?(Symbol) ||
+                       value.is_a?(FalseClass) ||
+                       value.is_a?(TrueClass) ||
+                       value.nil?
+
+        raise InvalidLabelError, "#{key} does not contain a valid value (type #{value.class})"
+      end
+    end
+  end
+end

data/lib/prometheus/client/metric.rb

@@ -0,0 +1,80 @@
+require 'thread'
+require 'prometheus/client/label_set_validator'
+require 'prometheus/client/uses_value_type'
+
+module Prometheus
+  module Client
+    class Metric
+      include UsesValueType
+      attr_reader :name, :docstring, :base_labels
+
+      def initialize(name, docstring, base_labels = {})
+        @mutex = Mutex.new
+        @validator = case type
+                       when :summary
+                         LabelSetValidator.new(['quantile'])
+                       when :histogram
+                         LabelSetValidator.new(['le'])
+                       else
+                         LabelSetValidator.new
+                     end
+        @values = Hash.new { |hash, key| hash[key] = default(key) }
+
+        validate_name(name)
+        validate_docstring(docstring)
+        @validator.valid?(base_labels)
+
+        @name = name
+        @docstring = docstring
+        @base_labels = base_labels
+      end
+
+      # Returns the value for the given label set
+      def get(labels = {})
+        label_set = label_set_for(labels)
+        @validator.valid?(label_set)
+
+        @values[label_set].get
+      end
+
+      # Returns all label sets with their values
+      def values
+        synchronize do
+          @values.each_with_object({}) do |(labels, value), memo|
+            memo[labels] = value
+          end
+        end
+      end
+
+      private
+
+      def touch_default_value
+        @values[label_set_for({})]
+      end
+
+      def default(labels)
+        value_object(type, @name, @name, labels)
+      end
+
+      def validate_name(name)
+        return true if name.is_a?(Symbol)
+
+        raise ArgumentError, 'given name must be a symbol'
+      end
+
+      def validate_docstring(docstring)
+        return true if docstring.respond_to?(:empty?) && !docstring.empty?
+
+        raise ArgumentError, 'docstring must be given'
+      end
+
+      def label_set_for(labels)
+        @validator.validate(@base_labels.merge(labels))
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+    end
+  end
+end