vinted-prometheus-client-mmap 1.5.0-x86_64-linux

data/ext/fast_mmaped_file_rs/src/testhelper.rs

@@ -0,0 +1,222 @@
+use bstr::{BString, B};
+use std::fs::File;
+use std::io::{Read, Seek, Write};
+use std::path::PathBuf;
+use tempfile::{tempdir, TempDir};
+
+use crate::raw_entry::RawEntry;
+use crate::HEADER_SIZE;
+
+#[derive(PartialEq, Default, Debug)]
+pub struct TestEntry {
+    pub header: u32,
+    pub json: &'static str,
+    pub padding_len: usize,
+    pub value: f64,
+}
+
+impl TestEntry {
+    pub fn new(json: &'static str, value: f64) -> Self {
+        TestEntry {
+            header: json.len() as u32,
+            json,
+            padding_len: RawEntry::padding_len(json.len()),
+            value,
+        }
+    }
+
+    pub fn as_bytes(&self) -> Vec<u8> {
+        [
+            B(&self.header.to_ne_bytes()),
+            self.json.as_bytes(),
+            &vec![b' '; self.padding_len],
+            B(&self.value.to_ne_bytes()),
+        ]
+        .concat()
+    }
+    pub fn as_bstring(&self) -> BString {
+        [
+            B(&self.header.to_ne_bytes()),
+            self.json.as_bytes(),
+            &vec![b' '; self.padding_len],
+            B(&self.value.to_ne_bytes()),
+        ]
+        .concat()
+        .into()
+    }
+
+    pub fn as_bytes_no_header(&self) -> BString {
+        [
+            self.json.as_bytes(),
+            &vec![b' '; self.padding_len],
+            B(&self.value.to_ne_bytes()),
+        ]
+        .concat()
+        .into()
+    }
+}
+
+/// Format the data for a `.db` file.
+/// Optional header value can be used to set an invalid `used` size.
+pub fn entries_to_db(entries: &[&'static str], values: &[f64], header: Option<u32>) -> Vec<u8> {
+    let mut out = Vec::new();
+
+    let entry_bytes: Vec<_> = entries
+        .iter()
+        .zip(values)
+        .flat_map(|(e, val)| TestEntry::new(e, *val).as_bytes())
+        .collect();
+
+    let used = match header {
+        Some(u) => u,
+        None => (entry_bytes.len() + HEADER_SIZE) as u32,
+    };
+
+    out.extend(used.to_ne_bytes());
+    out.extend([0x0u8; 4]); // Padding.
+    out.extend(entry_bytes);
+
+    out
+}
+
+/// A temporary file, path, and dir for use with testing.
+#[derive(Debug)]
+pub struct TestFile {
+    pub file: File,
+    pub path: PathBuf,
+    pub dir: TempDir,
+}
+
+impl TestFile {
+    pub fn new(file_data: &[u8]) -> TestFile {
+        let dir = tempdir().unwrap();
+        let path = dir.path().join("test.db");
+        let mut file = File::options()
+            .create(true)
+            .read(true)
+            .write(true)
+            .open(&path)
+            .unwrap();
+
+        file.write_all(file_data).unwrap();
+        file.sync_all().unwrap();
+        file.rewind().unwrap();
+
+        // We need to keep `dir` in scope so it doesn't drop before the files it
+        // contains, which may prevent cleanup.
+        TestFile { file, path, dir }
+    }
+}
+
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_entry_new() {
+        let json = "foobar";
+        let value = 1.0f64;
+        let expected = TestEntry {
+            header: 6,
+            json,
+            padding_len: 6,
+            value,
+        };
+
+        let actual = TestEntry::new(json, value);
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    fn test_entry_bytes() {
+        let json = "foobar";
+        let value = 1.0f64;
+        let expected = [
+            &6u32.to_ne_bytes(),
+            B(json),
+            &[b' '; 6],
+            &value.to_ne_bytes(),
+        ]
+        .concat();
+
+        let actual = TestEntry::new(json, value).as_bstring();
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    fn test_entry_bytes_no_header() {
+        let json = "foobar";
+        let value = 1.0f64;
+        let expected = [B(json), &[b' '; 6], &value.to_ne_bytes()].concat();
+
+        let actual = TestEntry::new(json, value).as_bytes_no_header();
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    fn test_entries_to_db_header_correct() {
+        let json = &["foobar", "qux"];
+        let values = &[1.0, 2.0];
+
+        let out = entries_to_db(json, values, None);
+
+        assert_eq!(48u32.to_ne_bytes(), out[0..4], "used set correctly");
+        assert_eq!([0u8; 4], out[4..8], "padding set");
+        assert_eq!(
+            TestEntry::new(json[0], values[0]).as_bytes(),
+            out[8..32],
+            "first entry matches"
+        );
+        assert_eq!(
+            TestEntry::new(json[1], values[1]).as_bytes(),
+            out[32..48],
+            "second entry matches"
+        );
+    }
+
+    #[test]
+    fn test_entries_to_db_header_wrong() {
+        let json = &["foobar", "qux"];
+        let values = &[1.0, 2.0];
+
+        const WRONG_USED: u32 = 1000;
+        let out = entries_to_db(json, values, Some(WRONG_USED));
+
+        assert_eq!(
+            WRONG_USED.to_ne_bytes(),
+            out[0..4],
+            "used set to value requested"
+        );
+        assert_eq!([0u8; 4], out[4..8], "padding set");
+        assert_eq!(
+            TestEntry::new(json[0], values[0]).as_bytes(),
+            out[8..32],
+            "first entry matches"
+        );
+        assert_eq!(
+            TestEntry::new(json[1], values[1]).as_bytes(),
+            out[32..48],
+            "second entry matches"
+        );
+    }
+
+    #[test]
+    fn test_file() {
+        let mut test_file = TestFile::new(b"foobar");
+        let stat = test_file.file.metadata().unwrap();
+
+        assert_eq!(6, stat.len(), "file length");
+        assert_eq!(
+            0,
+            test_file.file.stream_position().unwrap(),
+            "at start of file"
+        );
+        let mut out_buf = vec![0u8; 256];
+        let read_result = test_file.file.read(&mut out_buf);
+        assert!(read_result.is_ok());
+        assert_eq!(6, read_result.unwrap(), "file is readable");
+
+        let write_result = test_file.file.write(b"qux");
+        assert!(write_result.is_ok());
+        assert_eq!(3, write_result.unwrap(), "file is writable");
+    }
+}

data/ext/fast_mmaped_file_rs/src/util.rs

@@ -0,0 +1,140 @@
+use nix::errno::Errno;
+use nix::libc::c_long;
+use std::fmt::Display;
+use std::io;
+use std::mem::size_of;
+
+use crate::error::MmapError;
+use crate::exemplars::{Exemplar, EXEMPLAR_ENTRY_MAX_SIZE_BYTES};
+use crate::Result;
+
+/// Wrapper around `checked_add()` that converts failures
+/// to `MmapError::Overflow`.
+pub trait CheckedOps: Sized {
+    fn add_chk(self, rhs: Self) -> Result<Self>;
+    fn mul_chk(self, rhs: Self) -> Result<Self>;
+}
+
+impl CheckedOps for usize {
+    fn add_chk(self, rhs: Self) -> Result<Self> {
+        self.checked_add(rhs)
+            .ok_or_else(|| MmapError::overflowed(self, rhs, "adding"))
+    }
+
+    fn mul_chk(self, rhs: Self) -> Result<Self> {
+        self.checked_mul(rhs)
+            .ok_or_else(|| MmapError::overflowed(self, rhs, "multiplying"))
+    }
+}
+
+impl CheckedOps for c_long {
+    fn add_chk(self, rhs: Self) -> Result<Self> {
+        self.checked_add(rhs)
+            .ok_or_else(|| MmapError::overflowed(self, rhs, "adding"))
+    }
+
+    fn mul_chk(self, rhs: Self) -> Result<Self> {
+        self.checked_mul(rhs)
+            .ok_or_else(|| MmapError::overflowed(self, rhs, "multiplying"))
+    }
+}
+
+/// A wrapper around `TryFrom`, returning `MmapError::FailedCast` on error.
+pub fn cast_chk<T, U>(val: T, name: &str) -> Result<U>
+where
+    T: Copy + Display,
+    U: std::convert::TryFrom<T>,
+{
+    U::try_from(val).map_err(|_| MmapError::failed_cast::<T, U>(val, name))
+}
+
+/// Retrieve errno(3).
+pub fn errno() -> i32 {
+    // UNWRAP: This will always return `Some` when called from `last_os_error()`.
+    io::Error::last_os_error().raw_os_error().unwrap()
+}
+
+/// Get the error string associated with errno(3).
+/// Equivalent to strerror(3).
+pub fn strerror(errno: i32) -> &'static str {
+    Errno::from_i32(errno).desc()
+}
+
+/// Read a `u32` value from a byte slice starting from `offset`.
+#[inline]
+pub fn read_u32(buf: &[u8], offset: usize) -> Result<u32> {
+    if let Some(slice) = buf.get(offset..offset + size_of::<u32>()) {
+        // UNWRAP: We can safely unwrap the conversion from slice to array as we
+        // the source and targets are constructed here with the same length.
+        let out: &[u8; size_of::<u32>()] = slice.try_into().unwrap();
+
+        return Ok(u32::from_ne_bytes(*out));
+    }
+    Err(MmapError::out_of_bounds(offset, buf.len()))
+}
+
+/// Read an `f64` value from a byte slice starting from `offset`.
+#[inline]
+pub fn read_f64(buf: &[u8], offset: usize) -> Result<f64> {
+    if let Some(slice) = buf.get(offset..offset + size_of::<f64>()) {
+        // UNWRAP: We can safely unwrap the conversion from slice to array as we
+        // can be sure the target array has same length as the source slice.
+        let out: &[u8; size_of::<f64>()] = slice.try_into().unwrap();
+
+        return Ok(f64::from_ne_bytes(*out));
+    }
+    Err(MmapError::out_of_bounds(
+        offset + size_of::<f64>(),
+        buf.len(),
+    ))
+}
+
+pub fn read_exemplar(buf: &[u8], offset: usize) -> Result<Exemplar> {
+    if let Some(slice) = buf.get(offset..offset + EXEMPLAR_ENTRY_MAX_SIZE_BYTES) {
+        // UNWRAP: We can safely unwrap the conversion from slice to array as we
+       // can be sure the target array has same length as the source slice.
+       let out: &[u8; EXEMPLAR_ENTRY_MAX_SIZE_BYTES] = slice.try_into().expect("failed to convert slice to array");
+
+       let res: Vec<u8> = out.iter().cloned().filter(|&x| x != 0).collect();
+
+       let v: Exemplar = serde_json::from_slice(&res).expect("failed to convert string to Exemplar");
+        
+       return Ok(v)
+    }
+    Err(MmapError::out_of_bounds(
+        offset + EXEMPLAR_ENTRY_MAX_SIZE_BYTES,
+        buf.len(),
+    ))
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_read_u32() {
+        let buf = 1u32.to_ne_bytes();
+
+        assert!(matches!(read_u32(&buf, 0), Ok(1)), "index ok");
+        assert!(read_u32(&buf, 10).is_err(), "index out of range");
+        assert!(
+            read_u32(&buf, 1).is_err(),
+            "index in range but end out of range"
+        );
+    }
+
+    #[test]
+    fn test_read_f64() {
+        let buf = 1.00f64.to_ne_bytes();
+
+        let ok = read_f64(&buf, 0);
+        assert!(ok.is_ok());
+        assert_eq!(ok.unwrap(), 1.00);
+
+        assert!(read_f64(&buf, 10).is_err(), "index out of range");
+        assert!(
+            read_f64(&buf, 1).is_err(),
+            "index in range but end out of range"
+        );
+    }
+}

data/lib/prometheus/client/configuration.rb

@@ -0,0 +1,24 @@
+require 'prometheus/client/registry'
+require 'prometheus/client/mmaped_value'
+require 'prometheus/client/page_size'
+require 'logger'
+require 'tmpdir'
+
+module Prometheus
+  module Client
+    class Configuration
+      attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider, :enable_protobuf
+
+      def initialize
+        @value_class = ::Prometheus::Client::MmapedValue
+        @initial_mmap_file_size = ::Prometheus::Client::PageSize.page_size(fallback_page_size: 4096)
+        @logger = Logger.new($stdout)
+        @pid_provider = Process.method(:pid)
+        @multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do
+          Dir.mktmpdir("prometheus-mmap")
+        end
+        @enable_protobuf = false
+      end
+    end
+  end
+end

data/lib/prometheus/client/counter.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+require 'prometheus/client/metric'
+
+module Prometheus
+  module Client
+    # Counter is a metric that exposes merely a sum or tally of things.
+    class Counter < Metric
+      def type
+        :counter
+      end
+
+      def increment(labels = {}, by = 1, exemplar_name = '', exemplar_value = '')
+        raise ArgumentError, 'increment must be a non-negative number' if by < 0
+
+        label_set = label_set_for(labels)
+        synchronize { @values[label_set].increment(by, exemplar_name, exemplar_value) }
+      end
+
+      private
+
+      def default(labels)
+        value_object(type, @name, @name, labels)
+      end
+    end
+  end
+end

data/lib/prometheus/client/formats/protobuf.rb

@@ -0,0 +1,93 @@
+require 'prometheus/client/uses_value_type'
+require 'prometheus/client/helper/json_parser'
+require 'prometheus/client/helper/plain_file'
+require 'prometheus/client/helper/metrics_processing'
+require 'prometheus/client/helper/metrics_representation'
+
+module Prometheus
+  module Client
+    module Formats
+      # Protobuf format supports the same metric types as the text format plus
+      # native histograms. It is only available with the Rust extension.
+      module Protobuf
+        MEDIA_TYPE = 'application/vnd.google.protobuf'.freeze
+        PROTO = 'io.prometheus.client.MetricFamily'.freeze
+        ENCODING = 'delimited'.freeze
+        CONTENT_TYPE = "#{MEDIA_TYPE}; proto=#{PROTO}; encoding=#{ENCODING}".freeze
+
+        class << self
+          def marshal(registry)
+            metrics = registry.metrics.map do |metric|
+              samples = metric.values.flat_map do |label_set, value|
+                representation(metric, label_set, value)
+              end
+
+              [metric.name, { type: metric.type, help: metric.docstring, samples: samples }]
+            end
+
+            Helper::MetricsRepresentation.to_metrics(metrics)
+          end
+
+          def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir, use_rust: true)
+            # NOTE(GiedriusS): need to ensure exemplar files go at the end because they add extra data.
+            file_list = Dir.glob(File.join(path, '*.db')).sort_by { |f| [f.include?('exemplar') ? 1 : 0, f] }
+              .map {|f| Helper::PlainFile.new(f) }
+              .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
+
+            FastMmapedFileRs.to_protobuf(file_list.to_a)
+          end
+
+          def rust_impl_available?
+            return @rust_available unless @rust_available.nil?
+
+            check_for_rust
+          end
+
+          private
+
+          def load_metrics(path)
+            metrics = {}
+            Dir.glob(File.join(path, '*.db')).sort.each do |f|
+              Helper::PlainFile.new(f).to_metrics(metrics)
+            end
+
+            metrics
+          end
+
+          def representation(metric, label_set, value)
+            labels = metric.base_labels.merge(label_set)
+
+            if metric.type == :summary
+              summary(metric.name, labels, value)
+            elsif metric.type == :histogram
+              histogram(metric.name, labels, value)
+            else
+              [[metric.name, labels, value.get]]
+            end
+          end
+
+          def summary(name, set, value)
+            rv = value.get.map do |q, v|
+              [name, set.merge(quantile: q), v]
+            end
+
+            rv << ["#{name}_sum", set, value.get.sum]
+            rv << ["#{name}_count", set, value.get.total]
+            rv
+          end
+
+          def histogram(name, set, value)
+            # |metric_name, labels, value|
+            rv = value.get.map do |q, v|
+              [name, set.merge(le: q), v]
+            end
+
+            rv << ["#{name}_sum", set, value.get.sum]
+            rv << ["#{name}_count", set, value.get.total]
+            rv
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/formats/text.rb

@@ -0,0 +1,85 @@
+require 'prometheus/client/uses_value_type'
+require 'prometheus/client/helper/json_parser'
+require 'prometheus/client/helper/plain_file'
+require 'prometheus/client/helper/metrics_processing'
+require 'prometheus/client/helper/metrics_representation'
+
+module Prometheus
+  module Client
+    module Formats
+      # Text format is human readable mainly used for manual inspection.
+      module Text
+        MEDIA_TYPE = 'text/plain'.freeze
+        VERSION = '0.0.4'.freeze
+        CONTENT_TYPE = "#{MEDIA_TYPE}; version=#{VERSION}".freeze
+
+        class << self
+          def marshal(registry)
+            metrics = registry.metrics.map do |metric|
+              samples = metric.values.flat_map do |label_set, value|
+                representation(metric, label_set, value)
+              end
+
+              [metric.name, { type: metric.type, help: metric.docstring, samples: samples }]
+            end
+
+            Helper::MetricsRepresentation.to_text(metrics)
+          end
+
+          def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir)
+            file_list = Dir.glob(File.join(path, '*.db')).sort
+              .map {|f| Helper::PlainFile.new(f) }
+              .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
+
+              FastMmapedFileRs.to_metrics(file_list.to_a)
+          end
+
+          private
+
+          def load_metrics(path)
+            metrics = {}
+            Dir.glob(File.join(path, '*.db')).sort.each do |f|
+              Helper::PlainFile.new(f).to_metrics(metrics)
+            end
+
+            metrics
+          end
+
+          def representation(metric, label_set, value)
+            labels = metric.base_labels.merge(label_set)
+
+            if metric.type == :summary
+              summary(metric.name, labels, value)
+            elsif metric.type == :histogram
+              histogram(metric.name, labels, value)
+            else
+              [[metric.name, labels, value.get]]
+            end
+          end
+
+          def summary(name, set, value)
+            rv = value.get.map do |q, v|
+              [name, set.merge(quantile: q), v]
+            end
+
+            rv << ["#{name}_sum", set, value.get.sum]
+            rv << ["#{name}_count", set, value.get.total]
+            rv
+          end
+
+          def histogram(name, set, value)
+            # |metric_name, labels, value|
+            rv = value.get.map do |q, v|
+              [name, set.merge(le: q), v]
+            end
+
+            rv << [name, set.merge(le: '+Inf'), value.get.total]
+            rv << ["#{name}_sum", set, value.get.sum]
+            rv << ["#{name}_count", set, value.get.total]
+            rv
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/gauge.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+
+require 'prometheus/client/metric'
+
+module Prometheus
+  module Client
+    # A Gauge is a metric that exposes merely an instantaneous value or some
+    # snapshot thereof.
+    class Gauge < Metric
+      def initialize(name, docstring, base_labels = {}, multiprocess_mode=:all)
+        super(name, docstring, base_labels)
+        if value_class.multiprocess and ![:min, :max, :livesum, :liveall, :all].include?(multiprocess_mode)
+          raise ArgumentError, 'Invalid multiprocess mode: ' + multiprocess_mode
+        end
+        @multiprocess_mode = multiprocess_mode
+      end
+
+      def type
+        :gauge
+      end
+
+      def default(labels)
+        value_object(type, @name, @name, labels, @multiprocess_mode)
+      end
+
+      # Sets the value for the given label set
+      def set(labels, value)
+        @values[label_set_for(labels)].set(value)
+      end
+
+      def increment(labels, value)
+        @values[label_set_for(labels)].increment(value)
+      end
+
+      def decrement(labels, value)
+        @values[label_set_for(labels)].decrement(value)
+      end
+    end
+  end
+end