prometheus-client-mmap 0.20.3-x86_64-linux

data/ext/fast_mmaped_file_rs/src/util.rs

@@ -0,0 +1,121 @@
+use nix::errno::Errno;
+use nix::libc::c_long;
+use std::fmt::Display;
+use std::io;
+use std::mem::size_of;
+
+use crate::error::MmapError;
+use crate::Result;
+
+/// Wrapper around `checked_add()` that converts failures
+/// to `MmapError::Overflow`.
+pub trait CheckedOps: Sized {
+    fn add_chk(self, rhs: Self) -> Result<Self>;
+    fn mul_chk(self, rhs: Self) -> Result<Self>;
+}
+
+impl CheckedOps for usize {
+    fn add_chk(self, rhs: Self) -> Result<Self> {
+        self.checked_add(rhs)
+            .ok_or_else(|| MmapError::overflowed(self, rhs, "adding"))
+    }
+
+    fn mul_chk(self, rhs: Self) -> Result<Self> {
+        self.checked_mul(rhs)
+            .ok_or_else(|| MmapError::overflowed(self, rhs, "multiplying"))
+    }
+}
+
+impl CheckedOps for c_long {
+    fn add_chk(self, rhs: Self) -> Result<Self> {
+        self.checked_add(rhs)
+            .ok_or_else(|| MmapError::overflowed(self, rhs, "adding"))
+    }
+
+    fn mul_chk(self, rhs: Self) -> Result<Self> {
+        self.checked_mul(rhs)
+            .ok_or_else(|| MmapError::overflowed(self, rhs, "multiplying"))
+    }
+}
+
+/// A wrapper around `TryFrom`, returning `MmapError::FailedCast` on error.
+pub fn cast_chk<T, U>(val: T, name: &str) -> Result<U>
+where
+    T: Copy + Display,
+    U: std::convert::TryFrom<T>,
+{
+    U::try_from(val).map_err(|_| MmapError::failed_cast::<T, U>(val, name))
+}
+
+/// Retrieve errno(3).
+pub fn errno() -> i32 {
+    // UNWRAP: This will always return `Some` when called from `last_os_error()`.
+    io::Error::last_os_error().raw_os_error().unwrap()
+}
+
+/// Get the error string associated with errno(3).
+/// Equivalent to strerror(3).
+pub fn strerror(errno: i32) -> &'static str {
+    Errno::from_i32(errno).desc()
+}
+
+/// Read a `u32` value from a byte slice starting from `offset`.
+#[inline]
+pub fn read_u32(buf: &[u8], offset: usize) -> Result<u32> {
+    if let Some(slice) = buf.get(offset..offset + size_of::<u32>()) {
+        // UNWRAP: We can safely unwrap the conversion from slice to array as we
+        // the source and targets are constructed here with the same length.
+        let out: &[u8; size_of::<u32>()] = slice.try_into().unwrap();
+
+        return Ok(u32::from_ne_bytes(*out));
+    }
+    Err(MmapError::out_of_bounds(offset, buf.len()))
+}
+
+/// Read an `f64` value from a byte slice starting from `offset`.
+#[inline]
+pub fn read_f64(buf: &[u8], offset: usize) -> Result<f64> {
+    if let Some(slice) = buf.get(offset..offset + size_of::<f64>()) {
+        // UNWRAP: We can safely unwrap the conversion from slice to array as we
+        // can be sure the target array has same length as the source slice.
+        let out: &[u8; size_of::<f64>()] = slice.try_into().unwrap();
+
+        return Ok(f64::from_ne_bytes(*out));
+    }
+    Err(MmapError::out_of_bounds(
+        offset + size_of::<f64>(),
+        buf.len(),
+    ))
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_read_u32() {
+        let buf = 1u32.to_ne_bytes();
+
+        assert!(matches!(read_u32(&buf, 0), Ok(1)), "index ok");
+        assert!(read_u32(&buf, 10).is_err(), "index out of range");
+        assert!(
+            read_u32(&buf, 1).is_err(),
+            "index in range but end out of range"
+        );
+    }
+
+    #[test]
+    fn test_read_f64() {
+        let buf = 1.00f64.to_ne_bytes();
+
+        let ok = read_f64(&buf, 0);
+        assert!(ok.is_ok());
+        assert_eq!(ok.unwrap(), 1.00);
+
+        assert!(read_f64(&buf, 10).is_err(), "index out of range");
+        assert!(
+            read_f64(&buf, 1).is_err(),
+            "index in range but end out of range"
+        );
+    }
+}

data/lib/prometheus/client/configuration.rb

@@ -0,0 +1,23 @@
+require 'prometheus/client/registry'
+require 'prometheus/client/mmaped_value'
+require 'prometheus/client/page_size'
+require 'logger'
+require 'tmpdir'
+
+module Prometheus
+  module Client
+    class Configuration
+      attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider
+
+      def initialize
+        @value_class = ::Prometheus::Client::MmapedValue
+        @initial_mmap_file_size = ::Prometheus::Client::PageSize.page_size(fallback_page_size: 4096)
+        @logger = Logger.new($stdout)
+        @pid_provider = Process.method(:pid)
+        @multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do
+          Dir.mktmpdir("prometheus-mmap")
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/counter.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+require 'prometheus/client/metric'
+
+module Prometheus
+  module Client
+    # Counter is a metric that exposes merely a sum or tally of things.
+    class Counter < Metric
+      def type
+        :counter
+      end
+
+      def increment(labels = {}, by = 1)
+        raise ArgumentError, 'increment must be a non-negative number' if by < 0
+
+        label_set = label_set_for(labels)
+        synchronize { @values[label_set].increment(by) }
+      end
+
+      private
+
+      def default(labels)
+        value_object(type, @name, @name, labels)
+      end
+    end
+  end
+end

data/lib/prometheus/client/formats/text.rb

@@ -0,0 +1,118 @@
+require 'prometheus/client/uses_value_type'
+require 'prometheus/client/helper/json_parser'
+require 'prometheus/client/helper/plain_file'
+require 'prometheus/client/helper/metrics_processing'
+require 'prometheus/client/helper/metrics_representation'
+
+module Prometheus
+  module Client
+    module Formats
+      # Text format is human readable mainly used for manual inspection.
+      module Text
+        MEDIA_TYPE = 'text/plain'.freeze
+        VERSION = '0.0.4'.freeze
+        CONTENT_TYPE = "#{MEDIA_TYPE}; version=#{VERSION}".freeze
+
+        class << self
+          def marshal(registry)
+            metrics = registry.metrics.map do |metric|
+              samples = metric.values.flat_map do |label_set, value|
+                representation(metric, label_set, value)
+              end
+
+              [metric.name, { type: metric.type, help: metric.docstring, samples: samples }]
+            end
+
+            Helper::MetricsRepresentation.to_text(metrics)
+          end
+
+          def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir, use_rust: false)
+            file_list = Dir.glob(File.join(path, '*.db')).sort
+              .map {|f| Helper::PlainFile.new(f) }
+              .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
+
+            if use_rust && rust_impl_available?
+              FastMmapedFileRs.to_metrics(file_list.to_a)
+            else
+              FastMmapedFile.to_metrics(file_list.to_a)
+            end
+          end
+
+          def rust_impl_available?
+            return @rust_available unless @rust_available.nil?
+
+            check_for_rust
+          end
+
+          private
+
+          def load_rust_extension
+            begin
+              ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
+              require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
+            rescue LoadError
+              require 'fast_mmaped_file_rs'
+            end
+          end
+
+          def check_for_rust
+            # This will be evaluated on each invocation even with `||=` if
+            # `@rust_available` if false. Running a `require` statement is slow,
+            # so the `rust_impl_available?` method memoizes the result, external
+            # callers can only trigger this method a single time.
+            @rust_available = begin
+              load_rust_extension
+              true
+            rescue LoadError
+              Prometheus::Client.logger.info('FastMmapedFileRs unavailable')
+              false
+            end
+          end
+
+          def load_metrics(path)
+            metrics = {}
+            Dir.glob(File.join(path, '*.db')).sort.each do |f|
+              Helper::PlainFile.new(f).to_metrics(metrics)
+            end
+
+            metrics
+          end
+
+          def representation(metric, label_set, value)
+            labels = metric.base_labels.merge(label_set)
+
+            if metric.type == :summary
+              summary(metric.name, labels, value)
+            elsif metric.type == :histogram
+              histogram(metric.name, labels, value)
+            else
+              [[metric.name, labels, value.get]]
+            end
+          end
+
+          def summary(name, set, value)
+            rv = value.get.map do |q, v|
+              [name, set.merge(quantile: q), v]
+            end
+
+            rv << ["#{name}_sum", set, value.get.sum]
+            rv << ["#{name}_count", set, value.get.total]
+            rv
+          end
+
+          def histogram(name, set, value)
+            # |metric_name, labels, value|
+            rv = value.get.map do |q, v|
+              [name, set.merge(le: q), v]
+            end
+
+            rv << [name, set.merge(le: '+Inf'), value.get.total]
+            rv << ["#{name}_sum", set, value.get.sum]
+            rv << ["#{name}_count", set, value.get.total]
+            rv
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/gauge.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+
+require 'prometheus/client/metric'
+
+module Prometheus
+  module Client
+    # A Gauge is a metric that exposes merely an instantaneous value or some
+    # snapshot thereof.
+    class Gauge < Metric
+      def initialize(name, docstring, base_labels = {}, multiprocess_mode=:all)
+        super(name, docstring, base_labels)
+        if value_class.multiprocess and ![:min, :max, :livesum, :liveall, :all].include?(multiprocess_mode)
+          raise ArgumentError, 'Invalid multiprocess mode: ' + multiprocess_mode
+        end
+        @multiprocess_mode = multiprocess_mode
+      end
+
+      def type
+        :gauge
+      end
+
+      def default(labels)
+        value_object(type, @name, @name, labels, @multiprocess_mode)
+      end
+
+      # Sets the value for the given label set
+      def set(labels, value)
+        @values[label_set_for(labels)].set(value)
+      end
+
+      def increment(labels, value)
+        @values[label_set_for(labels)].increment(value)
+      end
+
+      def decrement(labels, value)
+        @values[label_set_for(labels)].decrement(value)
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/entry_parser.rb

@@ -0,0 +1,132 @@
+require 'prometheus/client/helper/json_parser'
+
+module Prometheus
+  module Client
+    module Helper
+      module EntryParser
+        class ParsingError < RuntimeError;
+        end
+
+        MINIMUM_SIZE = 8
+        START_POSITION = 8
+        VALUE_BYTES = 8
+        ENCODED_LENGTH_BYTES = 4
+
+        def used
+          slice(0..3).unpack('l')[0]
+        end
+
+        def parts
+          @parts ||= File.basename(filepath, '.db')
+                       .split('_')
+                       .map { |e| e.gsub(/-\d+$/, '') } # remove trailing -number
+        end
+
+        def type
+          parts[0].to_sym
+        end
+
+        def pid
+          (parts[2..-1] || []).join('_')
+        end
+
+        def multiprocess_mode
+          parts[1]
+        end
+
+        def empty?
+          size < MINIMUM_SIZE || used.zero?
+        end
+
+        def entries(ignore_errors = false)
+          return Enumerator.new {} if empty?
+
+          Enumerator.new do |yielder|
+            used_ = used # cache used to avoid unnecessary unpack operations
+
+            pos = START_POSITION # used + padding offset
+            while pos < used_ && pos < size && pos > 0
+              data = slice(pos..-1)
+              unless data
+                raise ParsingError, "data slice is nil at pos #{pos}" unless ignore_errors
+                pos += 8
+                next
+              end
+
+              encoded_len, first_encoded_bytes = data.unpack('LL')
+              if encoded_len.nil? || encoded_len.zero? || first_encoded_bytes.nil? || first_encoded_bytes.zero?
+                # do not parse empty data
+                pos += 8
+                next
+              end
+
+              entry_len = ENCODED_LENGTH_BYTES + encoded_len
+              padding_len = 8 - entry_len % 8
+
+              value_offset = entry_len + padding_len # align to 8 bytes
+              pos += value_offset
+
+              if value_offset > 0 && (pos + VALUE_BYTES) <= size # if positions are safe
+                yielder.yield data, encoded_len, value_offset, pos
+              else
+                raise ParsingError, "data slice is nil at pos #{pos}" unless ignore_errors
+              end
+              pos += VALUE_BYTES
+            end
+          end
+        end
+
+        def parsed_entries(ignore_errors = false)
+          result = entries(ignore_errors).map do |data, encoded_len, value_offset, _|
+            begin
+              encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset))
+              [encoded, value]
+            rescue ArgumentError => e
+              Prometheus::Client.logger.debug("Error processing data: #{bin_to_hex(data[0, 7])} len: #{encoded_len} value_offset: #{value_offset}")
+              raise ParsingError, e unless ignore_errors
+            end
+          end
+
+          result.reject!(&:nil?) if ignore_errors
+          result
+        end
+
+        def to_metrics(metrics = {}, ignore_errors = false)
+          parsed_entries(ignore_errors).each do |key, value|
+            begin
+              metric_name, name, labelnames, labelvalues = JsonParser.load(key)
+              labelnames ||= []
+              labelvalues ||= []
+
+              metric = metrics.fetch(metric_name,
+                                     metric_name: metric_name,
+                                     help: 'Multiprocess metric',
+                                     type: type,
+                                     samples: [])
+              if type == :gauge
+                metric[:multiprocess_mode] = multiprocess_mode
+                metric[:samples] += [[name, labelnames.zip(labelvalues) + [['pid', pid]], value]]
+              else
+                # The duplicates and labels are fixed in the next for.
+                metric[:samples] += [[name, labelnames.zip(labelvalues), value]]
+              end
+              metrics[metric_name] = metric
+
+            rescue JSON::ParserError => e
+              raise ParsingError(e) unless ignore_errors
+            end
+          end
+
+          metrics.reject! { |e| e.nil? } if ignore_errors
+          metrics
+        end
+
+        private
+
+        def bin_to_hex(s)
+          s.each_byte.map { |b| b.to_s(16) }.join
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/file_locker.rb

@@ -0,0 +1,50 @@
+module Prometheus
+  module Client
+    module Helper
+      class FileLocker
+        class << self
+          LOCK_FILE_MUTEX = Mutex.new
+
+          def lock_to_process(filepath)
+            LOCK_FILE_MUTEX.synchronize do
+              @file_locks ||= {}
+              return false if @file_locks[filepath]
+
+              file = File.open(filepath, 'ab')
+              if file.flock(File::LOCK_NB | File::LOCK_EX)
+                @file_locks[filepath] = file
+                return true
+              else
+                return false
+              end
+            end
+          end
+
+          def unlock(filepath)
+            LOCK_FILE_MUTEX.synchronize do
+              @file_locks ||= {}
+              return false unless @file_locks[filepath]
+
+              file = @file_locks[filepath]
+              file.flock(File::LOCK_UN)
+              file.close
+              @file_locks.delete(filepath)
+            end
+          end
+
+          def unlock_all
+            LOCK_FILE_MUTEX.synchronize do
+              @file_locks ||= {}
+              @file_locks.values.each do |file|
+                file.flock(File::LOCK_UN)
+                file.close
+              end
+
+              @file_locks = {}
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/json_parser.rb

@@ -0,0 +1,23 @@
+require 'json'
+
+module Prometheus
+  module Client
+    module Helper
+      module JsonParser
+        class << self
+          if defined?(Oj)
+            def load(s)
+              Oj.load(s)
+            rescue Oj::ParseError, EncodingError => e
+              raise JSON::ParserError.new(e.message)
+            end
+          else
+            def load(s)
+              JSON.parse(s)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/metrics_processing.rb

@@ -0,0 +1,45 @@
+module Prometheus
+  module Client
+    module Helper
+      module MetricsProcessing
+        def self.merge_metrics(metrics)
+          metrics.each_value do |metric|
+            metric[:samples] = merge_samples(metric[:samples], metric[:type], metric[:multiprocess_mode]).map do |(name, labels), value|
+              [name, labels.to_h, value]
+            end
+          end
+        end
+
+        def self.merge_samples(raw_samples, metric_type, multiprocess_mode)
+          samples = {}
+          raw_samples.each do |name, labels, value|
+            without_pid = labels.reject { |l| l[0] == 'pid' }
+
+            case metric_type
+              when :gauge
+                case multiprocess_mode
+                  when 'min'
+                    s = samples.fetch([name, without_pid], value)
+                    samples[[name, without_pid]] = [s, value].min
+                  when 'max'
+                    s = samples.fetch([name, without_pid], value)
+                    samples[[name, without_pid]] = [s, value].max
+                  when 'livesum'
+                    s = samples.fetch([name, without_pid], 0.0)
+                    samples[[name, without_pid]] = s + value
+                  else # all/liveall
+                    samples[[name, labels]] = value
+                end
+              else
+                # Counter, Histogram and Summary.
+                s = samples.fetch([name, without_pid], 0.0)
+                samples[[name, without_pid]] = s + value
+            end
+          end
+
+          samples
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/metrics_representation.rb

@@ -0,0 +1,51 @@
+module Prometheus
+  module Client
+    module Helper
+      module MetricsRepresentation
+        METRIC_LINE = '%s%s %s'.freeze
+        TYPE_LINE = '# TYPE %s %s'.freeze
+        HELP_LINE = '# HELP %s %s'.freeze
+
+        LABEL = '%s="%s"'.freeze
+        SEPARATOR = ','.freeze
+        DELIMITER = "\n".freeze
+
+        REGEX = { doc: /[\n\\]/, label: /[\n\\"]/ }.freeze
+        REPLACE = { "\n" => '\n', '\\' => '\\\\', '"' => '\"' }.freeze
+
+        def self.to_text(metrics)
+          lines = []
+
+          metrics.each do |name, metric|
+            lines << format(HELP_LINE, name, escape(metric[:help]))
+            lines << format(TYPE_LINE, name, metric[:type])
+            metric[:samples].each do |metric_name, labels, value|
+              lines << metric(metric_name, format_labels(labels), value)
+            end
+          end
+
+          # there must be a trailing delimiter
+          (lines << nil).join(DELIMITER)
+        end
+
+        def self.metric(name, labels, value)
+          format(METRIC_LINE, name, labels, value)
+        end
+
+        def self.format_labels(set)
+          return if set.empty?
+
+          strings = set.each_with_object([]) do |(key, value), memo|
+            memo << format(LABEL, key, escape(value, :label))
+          end
+
+          "{#{strings.join(SEPARATOR)}}"
+        end
+
+        def self.escape(string, format = :doc)
+          string.to_s.gsub(REGEX[format], REPLACE)
+        end
+      end
+    end
+  end
+end