vinted-prometheus-client-mmap 1.2.1

data/lib/prometheus/client/helper/metrics_representation.rb

@@ -0,0 +1,51 @@
+module Prometheus
+  module Client
+    module Helper
+      module MetricsRepresentation
+        METRIC_LINE = '%s%s %s'.freeze
+        TYPE_LINE = '# TYPE %s %s'.freeze
+        HELP_LINE = '# HELP %s %s'.freeze
+
+        LABEL = '%s="%s"'.freeze
+        SEPARATOR = ','.freeze
+        DELIMITER = "\n".freeze
+
+        REGEX = { doc: /[\n\\]/, label: /[\n\\"]/ }.freeze
+        REPLACE = { "\n" => '\n', '\\' => '\\\\', '"' => '\"' }.freeze
+
+        def self.to_text(metrics)
+          lines = []
+
+          metrics.each do |name, metric|
+            lines << format(HELP_LINE, name, escape(metric[:help]))
+            lines << format(TYPE_LINE, name, metric[:type])
+            metric[:samples].each do |metric_name, labels, value|
+              lines << metric(metric_name, format_labels(labels), value)
+            end
+          end
+
+          # there must be a trailing delimiter
+          (lines << nil).join(DELIMITER)
+        end
+
+        def self.metric(name, labels, value)
+          format(METRIC_LINE, name, labels, value)
+        end
+
+        def self.format_labels(set)
+          return if set.empty?
+
+          strings = set.each_with_object([]) do |(key, value), memo|
+            memo << format(LABEL, key, escape(value, :label))
+          end
+
+          "{#{strings.join(SEPARATOR)}}"
+        end
+
+        def self.escape(string, format = :doc)
+          string.to_s.gsub(REGEX[format], REPLACE)
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/mmaped_file.rb

@@ -0,0 +1,64 @@
+require 'prometheus/client/helper/entry_parser'
+require 'prometheus/client/helper/file_locker'
+
+# load precompiled extension if available
+begin
+  ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
+  require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
+rescue LoadError
+  require 'fast_mmaped_file_rs'
+end
+
+module Prometheus
+  module Client
+    module Helper
+      class MmapedFile < FastMmapedFileRs
+        include EntryParser
+
+        attr_reader :filepath, :size
+
+        def initialize(filepath)
+          @filepath = filepath
+
+          File.open(filepath, 'a+b') do |file|
+            file.truncate(initial_mmap_file_size) if file.size < MINIMUM_SIZE
+            @size = file.size
+          end
+
+          super(filepath)
+        end
+
+        def close
+          munmap
+          FileLocker.unlock(filepath)
+        end
+
+        private
+
+        def initial_mmap_file_size
+          Prometheus::Client.configuration.initial_mmap_file_size
+        end
+
+        public
+
+        class << self
+          def open(filepath)
+            MmapedFile.new(filepath)
+          end
+
+          def ensure_exclusive_file(file_prefix = 'mmaped_file')
+            (0..Float::INFINITY).lazy
+              .map { |f_num| "#{file_prefix}_#{Prometheus::Client.pid}-#{f_num}.db" }
+              .map { |filename| File.join(Prometheus::Client.configuration.multiprocess_files_dir, filename) }
+              .find { |path| Helper::FileLocker.lock_to_process(path) }
+          end
+
+          def open_exclusive_file(file_prefix = 'mmaped_file')
+            filename = Helper::MmapedFile.ensure_exclusive_file(file_prefix)
+            open(filename)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/plain_file.rb

@@ -0,0 +1,29 @@
+require 'prometheus/client/helper/entry_parser'
+
+module Prometheus
+  module Client
+    module Helper
+      # Parses DB files without using mmap
+      class PlainFile
+        include EntryParser
+        attr_reader :filepath
+
+        def source
+          @data ||= File.read(filepath, mode: 'rb')
+        end
+
+        def initialize(filepath)
+          @filepath = filepath
+        end
+
+        def slice(*args)
+          source.slice(*args)
+        end
+
+        def size
+          source.length
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/histogram.rb

@@ -0,0 +1,80 @@
+require 'prometheus/client/metric'
+require 'prometheus/client/uses_value_type'
+
+module Prometheus
+  module Client
+    # A histogram samples observations (usually things like request durations
+    # or response sizes) and counts them in configurable buckets. It also
+    # provides a sum of all observed values.
+    class Histogram < Metric
+      # Value represents the state of a Histogram at a given point.
+      class Value < Hash
+        include UsesValueType
+        attr_accessor :sum, :total, :total_inf
+
+        def initialize(type, name, labels, buckets)
+          @sum = value_object(type, name, "#{name}_sum", labels)
+          @total = value_object(type, name, "#{name}_count", labels)
+          @total_inf = value_object(type, name, "#{name}_bucket", labels.merge(le: "+Inf"))
+
+          buckets.each do |bucket|
+            self[bucket] = value_object(type, name, "#{name}_bucket", labels.merge(le: bucket.to_s))
+          end
+        end
+
+        def observe(value, exemplar_name = '', exemplar_value = '')
+          @sum.increment(value, exemplar_name, exemplar_value)
+          @total.increment(1, exemplar_name, exemplar_value)
+          @total_inf.increment(1, exemplar_name, exemplar_value)
+
+          each_key do |bucket|
+            self[bucket].increment(1, exemplar_name, exemplar_value) if value <= bucket
+          end
+        end
+
+        def get()
+          hash = {}
+          each_key do |bucket|
+            hash[bucket] = self[bucket].get()
+          end
+          hash
+        end
+      end
+
+      # DEFAULT_BUCKETS are the default Histogram buckets. The default buckets
+      # are tailored to broadly measure the response time (in seconds) of a
+      # network service. (From DefBuckets client_golang)
+      DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1,
+                         2.5, 5, 10].freeze
+
+      # Offer a way to manually specify buckets
+      def initialize(name, docstring, base_labels = {},
+                     buckets = DEFAULT_BUCKETS)
+        raise ArgumentError, 'Unsorted buckets, typo?' unless sorted? buckets
+
+        @buckets = buckets
+        super(name, docstring, base_labels)
+      end
+
+      def type
+        :histogram
+      end
+
+      def observe(labels, value)
+        label_set = label_set_for(labels)
+        synchronize { @values[label_set].observe(value) }
+      end
+
+      private
+
+      def default(labels)
+        # TODO: default function needs to know key of hash info (label names and values)
+        Value.new(type, @name, labels, @buckets)
+      end
+
+      def sorted?(bucket)
+        bucket.each_cons(2).all? { |i, j| i <= j }
+      end
+    end
+  end
+end

data/lib/prometheus/client/label_set_validator.rb

@@ -0,0 +1,85 @@
+# encoding: UTF-8
+
+module Prometheus
+  module Client
+    # LabelSetValidator ensures that all used label sets comply with the
+    # Prometheus specification.
+    class LabelSetValidator
+      RESERVED_LABELS = [].freeze
+
+      class LabelSetError < StandardError; end
+      class InvalidLabelSetError < LabelSetError; end
+      class InvalidLabelError < LabelSetError; end
+      class ReservedLabelError < LabelSetError; end
+
+      def initialize(reserved_labels = [])
+        @reserved_labels = (reserved_labels + RESERVED_LABELS).freeze
+        @validated = {}
+      end
+
+      def valid?(labels)
+        unless labels.is_a?(Hash)
+          raise InvalidLabelSetError, "#{labels} is not a valid label set"
+        end
+
+        labels.all? do |key, value|
+          validate_symbol(key)
+          validate_name(key)
+          validate_reserved_key(key)
+          validate_value(key, value)
+        end
+      end
+
+      def validate(labels)
+        return labels if @validated.key?(labels.hash)
+
+        valid?(labels)
+
+        unless @validated.empty? || match?(labels, @validated.first.last)
+          raise InvalidLabelSetError, "labels must have the same signature: (#{label_diff(labels, @validated.first.last)})"
+        end
+
+        @validated[labels.hash] = labels
+      end
+
+      private
+
+      def label_diff(a, b)
+        "expected keys: #{b.keys.sort}, got: #{a.keys.sort}"
+      end
+
+      def match?(a, b)
+        a.keys.sort == b.keys.sort
+      end
+
+      def validate_symbol(key)
+        return true if key.is_a?(Symbol)
+
+        raise InvalidLabelError, "label #{key} is not a symbol"
+      end
+
+      def validate_name(key)
+        return true unless key.to_s.start_with?('__')
+
+        raise ReservedLabelError, "label #{key} must not start with __"
+      end
+
+      def validate_reserved_key(key)
+        return true unless @reserved_labels.include?(key)
+
+        raise ReservedLabelError, "#{key} is reserved"
+      end
+
+      def validate_value(key, value)
+        return true if value.is_a?(String) ||
+                       value.is_a?(Numeric) ||
+                       value.is_a?(Symbol) ||
+                       value.is_a?(FalseClass) ||
+                       value.is_a?(TrueClass) ||
+                       value.nil?
+
+        raise InvalidLabelError, "#{key} does not contain a valid value (type #{value.class})"
+      end
+    end
+  end
+end

data/lib/prometheus/client/metric.rb

@@ -0,0 +1,80 @@
+require 'thread'
+require 'prometheus/client/label_set_validator'
+require 'prometheus/client/uses_value_type'
+
+module Prometheus
+  module Client
+    class Metric
+      include UsesValueType
+      attr_reader :name, :docstring, :base_labels
+
+      def initialize(name, docstring, base_labels = {})
+        @mutex = Mutex.new
+        @validator = case type
+                       when :summary
+                         LabelSetValidator.new(['quantile'])
+                       when :histogram
+                         LabelSetValidator.new(['le'])
+                       else
+                         LabelSetValidator.new
+                     end
+        @values = Hash.new { |hash, key| hash[key] = default(key) }
+
+        validate_name(name)
+        validate_docstring(docstring)
+        @validator.valid?(base_labels)
+
+        @name = name
+        @docstring = docstring
+        @base_labels = base_labels
+      end
+
+      # Returns the value for the given label set
+      def get(labels = {})
+        label_set = label_set_for(labels)
+        @validator.valid?(label_set)
+
+        @values[label_set].get
+      end
+
+      # Returns all label sets with their values
+      def values
+        synchronize do
+          @values.each_with_object({}) do |(labels, value), memo|
+            memo[labels] = value
+          end
+        end
+      end
+
+      private
+
+      def touch_default_value
+        @values[label_set_for({})]
+      end
+
+      def default(labels)
+        value_object(type, @name, @name, labels)
+      end
+
+      def validate_name(name)
+        return true if name.is_a?(Symbol)
+
+        raise ArgumentError, 'given name must be a symbol'
+      end
+
+      def validate_docstring(docstring)
+        return true if docstring.respond_to?(:empty?) && !docstring.empty?
+
+        raise ArgumentError, 'docstring must be given'
+      end
+
+      def label_set_for(labels)
+        @validator.validate(@base_labels.merge(labels))
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+    end
+  end
+end

data/lib/prometheus/client/mmaped_dict.rb

@@ -0,0 +1,79 @@
+require 'prometheus/client/helper/mmaped_file'
+require 'prometheus/client/helper/plain_file'
+require 'prometheus/client'
+
+module Prometheus
+  module Client
+    class ParsingError < StandardError
+    end
+
+    # A dict of doubles, backed by an mmapped file.
+    #
+    # The file starts with a 4 byte int, indicating how much of it is used.
+    # Then 4 bytes of padding.
+    # There's then a number of entries, consisting of a 4 byte int which is the
+    # size of the next field, a utf-8 encoded string key, padding to an 8 byte
+    # alignment, and then a 8 byte float which is the value.
+    class MmapedDict
+      attr_reader :m, :used, :positions
+
+      def self.read_all_values(f)
+        Helper::PlainFile.new(f).entries.map do |data, encoded_len, value_offset, _|
+          encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset))
+          [encoded, value]
+        end
+      end
+
+      def initialize(m)
+        @mutex = Mutex.new
+
+        @m = m
+        # @m.mlock # TODO: Ensure memory is locked to RAM
+
+        @positions = {}
+        read_all_positions.each do |key, pos|
+          @positions[key] = pos
+        end
+      rescue StandardError => e
+        raise ParsingError, "exception #{e} while processing metrics file #{path}"
+      end
+
+      def read_value(key)
+        @m.fetch_entry(@positions, key, 0.0)
+      end
+
+      def write_value(key, value)
+        @m.upsert_entry(@positions, key, value)
+      end
+
+      def path
+        @m.filepath if @m
+      end
+
+      def close
+        @m.sync
+        @m.close
+      rescue TypeError => e
+        Prometheus::Client.logger.warn("munmap raised error #{e}")
+      end
+
+      def inspect
+        "#<#{self.class}:0x#{(object_id << 1).to_s(16)}>"
+      end
+
+      private
+
+      def init_value(key)
+        @m.add_entry(@positions, key, 0.0)
+      end
+
+      # Yield (key, pos). No locking is performed.
+      def read_all_positions
+        @m.entries.map do |data, encoded_len, _, absolute_pos|
+          encoded, = data.unpack(format('@4A%d', encoded_len))
+          [encoded, absolute_pos]
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/mmaped_value.rb

@@ -0,0 +1,158 @@
+require 'prometheus/client'
+require 'prometheus/client/mmaped_dict'
+require 'json'
+
+module Prometheus
+  module Client
+    # A float protected by a mutex backed by a per-process mmaped file.
+    class MmapedValue
+      VALUE_LOCK = Mutex.new
+
+      @@files = {}
+      @@pid = -1
+
+      def initialize(type, metric_name, name, labels, multiprocess_mode = '')
+        @file_prefix = type.to_s
+        @metric_name = metric_name
+        @name = name
+        @labels = labels
+        if type == :gauge
+          @file_prefix += '_' + multiprocess_mode.to_s
+        end
+
+        @pid = -1
+
+        @mutex = Mutex.new
+        initialize_file
+      end
+
+      def increment(amount = 1, exemplar_name = '', exemplar_value = '')
+        @mutex.synchronize do
+          initialize_file if pid_changed?
+
+          @value += amount
+          # TODO(GiedriusS): write exemplars too.
+          if @file_prefix != 'gauge'
+            puts "#{@name} exemplar name = #{exemplar_name}, exemplar_value = #{exemplar_value}"
+          end
+          write_value(@key, @value)
+          @value
+        end
+      end
+
+      def decrement(amount = 1)
+        increment(-amount)
+      end
+
+      def set(value)
+        @mutex.synchronize do
+          initialize_file if pid_changed?
+
+          @value = value
+          write_value(@key, @value)
+          @value
+        end
+      end
+
+      def get
+        @mutex.synchronize do
+          initialize_file if pid_changed?
+          return @value
+        end
+      end
+
+      def pid_changed?
+        @pid != Process.pid
+      end
+
+      # method needs to be run in VALUE_LOCK mutex
+      def unsafe_reinitialize_file(check_pid = true)
+        unsafe_initialize_file if !check_pid || pid_changed?
+      end
+
+      def self.reset_and_reinitialize
+        VALUE_LOCK.synchronize do
+          @@pid = Process.pid
+          @@files = {}
+
+          ObjectSpace.each_object(MmapedValue).each do |v|
+            v.unsafe_reinitialize_file(false)
+          end
+        end
+      end
+
+      def self.reset_on_pid_change
+        if pid_changed?
+          @@pid = Process.pid
+          @@files = {}
+        end
+      end
+
+      def self.reinitialize_on_pid_change
+        VALUE_LOCK.synchronize do
+          reset_on_pid_change
+
+          ObjectSpace.each_object(MmapedValue, &:unsafe_reinitialize_file)
+        end
+      end
+
+      def self.pid_changed?
+        @@pid != Process.pid
+      end
+
+      def self.multiprocess
+        true
+      end
+
+      private
+
+      def initialize_file
+        VALUE_LOCK.synchronize do
+          unsafe_initialize_file
+        end
+      end
+
+      def unsafe_initialize_file
+        self.class.reset_on_pid_change
+
+        @pid = Process.pid
+        unless @@files.has_key?(@file_prefix)
+          unless @file.nil?
+            @file.close
+          end
+          mmaped_file = Helper::MmapedFile.open_exclusive_file(@file_prefix)
+
+          @@files[@file_prefix] = MmapedDict.new(mmaped_file)
+        end
+
+        @file = @@files[@file_prefix]
+        @key = rebuild_key
+
+        @value = read_value(@key)
+      end
+
+
+      def rebuild_key
+        keys = @labels.keys.sort
+        values = @labels.values_at(*keys)
+
+        [@metric_name, @name, keys, values].to_json
+      end
+
+      def write_value(key, val)
+        @file.write_value(key, val)
+      rescue StandardError => e
+        Prometheus::Client.logger.warn("writing value to #{@file.path} failed with #{e}")
+        Prometheus::Client.logger.debug(e.backtrace.join("\n"))
+      end
+
+      def read_value(key)
+        @file.read_value(key)
+      rescue StandardError => e
+        Prometheus::Client.logger.warn("reading value from #{@file.path} failed with #{e}")
+        Prometheus::Client.logger.debug(e.backtrace.join("\n"))
+        0
+      end
+    end
+  end
+end

data/lib/prometheus/client/page_size.rb

@@ -0,0 +1,17 @@
+require 'open3'
+
+module Prometheus
+  module Client
+    module PageSize
+      def self.page_size(fallback_page_size: 4096)
+        stdout, status = Open3.capture2('getconf PAGESIZE')
+        return fallback_page_size if status.nil? || !status.success?
+
+        page_size = stdout.chomp.to_i
+        return fallback_page_size if page_size <= 0
+
+        page_size
+      end
+    end
+  end
+end