prometheus-client-mmap 0.22.0-x86_64-linux-musl → 1.2.1-x86_64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7413feb908630e80a9949470f32d3d93ad796b50e67dbdf4c601ff3254fc442e
-  data.tar.gz: 9f7d05d315e5ef4a4b776fd9e704599074756a8876bb2b22a44960571bda51d4
+  metadata.gz: 789dc7485bf3e87ace45cd1c90f63499f8edec0d01526a99198da98260b7b87c
+  data.tar.gz: 74b69f87d4401ba7027197ca74d5ad512413bf4f05224c552d4b4781b64dd1ac
 SHA512:
-  metadata.gz: 524070df387d7dbb72cf119715106b5586794263d433ed71b0e8da75a67147eeeddf365b587a6d9b13d26058d70600925ba037c2130bf87634b68baa06b4f52f
-  data.tar.gz: dcbb9b4f11ba3cd9ac67682c6a8f4462a920ff73b14f6578cd51c75d461cdd3cbcf433a5a1b21398e3c3bbb81131af5b9d77023adf50ba64a13fadb1e530181b
+  metadata.gz: dbbbf7a942c902dd3d884e3ff4e50212c5f0fa8f366b827e2ace2d372308f673a9e7557ea5e8f1c821241b698fbcd8aa1efdc72f106cea276dfd4f000fe35ab9
+  data.tar.gz: eb0a13f5f47cbc321504d6c20dcd096151dd52950f54e4a006dc7e5bf301a34881ef523bf11a56e0eab93b76f91a42efe309c69246ceae984ffee4892413893a

data/.tool-versions

@@ -0,0 +1 @@
+rust 1.73.0

data/README.md

@@ -10,7 +10,24 @@ through a HTTP interface. Intended to be used together with a
 
 [![Gem Version][4]](http://badge.fury.io/rb/prometheus-client-mmap)
 [![Build Status][3]](https://gitlab.com/gitlab-org/prometheus-client-mmap/commits/master)
-[![Dependency Status][5]](https://gemnasium.com/prometheus/prometheus-client-mmap)
+
+## Installation
+
+prometheus-client-mmap ships with [precompiled native gems](https://rubygems.org/gems/prometheus-client-mmap).
+Install the gem via:
+
+```shell
+gem install prometheus-client-mmap
+```
+
+### Requirements for building from source
+
+Building from source requires a number of dependencies:
+
+* Rust v1.65+
+* clang 5.0 or higher for [bindgen](https://rust-lang.github.io/rust-bindgen/requirements.html)
+* `make`
+* A C compiler (for legacy extension, which will be removed soon)
 
 ## Usage
 
@@ -34,19 +51,11 @@ http_requests = prometheus.counter(:http_requests, 'A counter of HTTP requests m
 http_requests.increment
 ```
 
-## Rust extension (experimental)
+## Rust extension
 
-In an effort to improve maintainability, there is now an optional Rust
+This gem now uses a rewritten Rust extension instead of C.
 implementation that reads the metric files and outputs the multiprocess
-metrics to text. If `rustc` is available, then the Rust extension will
-be built automatically. The `use_rust` keyword argument can be used:
-
-```ruby
-puts Prometheus::Client::Formats::Text.marshal_multiprocess(use_rust: true)
-```
-
-Note that this parameter will likely be deprecated and removed once the Rust
-extension becomes the default mode.
+metrics to text. This implementation is significantly faster than the C extension.
 
 ### Rack middleware
 
@@ -204,8 +213,8 @@ prometheus_multiproc_dir=/tmp
 
 ### PID cardinality
 
-In multiprocess setup e.g. running under Unicorn, having worker process restart often can
-lead to performance problems when proccesing metric files. By default each process using
+In multiprocess setup e.g. running under Unicorn or Puma, having worker process restart often
+can lead to performance problems when proccesing metric files. By default each process using
 Prometheus metrics will create a set of files based on that process PID. With high worker
 churn this will lead to creation of thousands of files and in turn will cause very noticable
 slowdown when displaying metrics
@@ -213,17 +222,23 @@ slowdown when displaying metrics
 To reduce this problem, a surrogate process id can be used. Set of all such IDs needs
 have low cardinality, and each process id must be unique among all running process.
 
-For Unicorn a worker id/number can be used to greatly speedup the metrics rendering.
+For Unicorn and Puma a worker id/number can be used to greatly speedup the metrics rendering.
 
-To use it add this line to your `configure` block:
+If you are using Unicorn, add this line to your `configure` block:
 
 ```ruby
   config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider)
 ```
 
+If you are using Puma, add this line to your `configure` block:
+
+```ruby
+  config.pid_provider = Prometheus::Client::Support::Puma.method(:worker_pid_provider)
+```
+
 ## Tools
 
-###`bin/parse`
+### `bin/parse`
 
 This command can be used to parse metric files located on the filesystem just like a metric exporter would.
 It outputs either `json` formatted raw data or digested data in prometheus `text` format.

data/ext/fast_mmaped_file_rs/Cargo.toml

@@ -6,24 +6,29 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-hashbrown = "0.13"
+hashbrown = "0.14"
 libc = "0.2"
-magnus = { git = "https://github.com/matsadler/magnus", branch = "main", features = ["rb-sys"] }
-memmap2 = "0.5"
+magnus = { version = "0.7", features = ["rb-sys"] }
+memmap2 = "0.9"
 # v0.26 cannot be built on CentOS 7 https://github.com/nix-rust/nix/issues/1972
 nix = { version = "0.25", features = ["mman"] } # mman used for MsFlags
-rb-sys = "0.9"
-smallvec = "1.10"
-thiserror = "1.0"
+rb-sys = { version = "0.9", features = ["stable-api-compiled-fallback"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0", features = ["raw_value"] }
+smallvec = { version = "1.13", features = ["serde"] }
+thiserror = "2.0"
 
 [dev-dependencies]
-bstr = "1.4"
+bstr = "1.11"
 indoc = "2.0"
 # We need the `embed` feature to run tests, but this triggers failures when building as a Gem.
-magnus = { git = "https://github.com/matsadler/magnus", branch = "main", features = ["rb-sys","embed"] }
+magnus = { version = "0.7", features = ["rb-sys","embed"] }
 rand = "0.8"
 sha2 = "0.10"
-tempfile = "3.5"
+tempfile = "3.14"
+
+[build-dependencies]
+rb-sys-env = { git = "https://github.com/oxidize-rb/rb-sys.git", rev = "5d1999ba70d301581524b8252c8ff3b8a825afa2" }
 
 [lib]
 # Integration tests won't work if crate is only `cdylib`.

data/ext/fast_mmaped_file_rs/build.rs

@@ -0,0 +1,5 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _ = rb_sys_env::activate()?;
+
+    Ok(())
+}

data/ext/fast_mmaped_file_rs/extconf.rb

@@ -24,7 +24,5 @@ if find_executable('rustc')
     ARGV = trimmed_argv
   end
 else
-  puts 'rustc not found, skipping Rust extension.'
-
-  File.write('Makefile', dummy_makefile($srcdir).join(''))
+  raise 'rustc not found. prometheus-client-mmap now requires Rust.'
 end

data/ext/fast_mmaped_file_rs/src/error.rs

@@ -9,7 +9,7 @@ use crate::util;
 use crate::PROM_EPARSING_ERROR;
 
 /// A lightweight representation of Ruby ExceptionClasses.
-#[derive(PartialEq, Clone, Copy, Debug)]
+#[derive(PartialEq, Eq, Clone, Copy, Debug)]
 pub enum RubyError {
     Arg,
     Encoding,
@@ -45,7 +45,7 @@ impl From<RubyError> for magnus::ExceptionClass {
 /// Errors returned internally within the crate. Methods called directly by Ruby return
 /// `magnus::error::Error` as do functions that interact heavily with Ruby. This can be
 /// converted into a `magnus::error::Error` at the boundary between Rust and Ruby.
-#[derive(PartialEq, Error, Debug)]
+#[derive(PartialEq, Eq, Error, Debug)]
 pub enum MmapError {
     /// A read or write was made while another thread had mutable access to the mmap.
     #[error("read/write operation attempted while mmap was being written to")]

data/ext/fast_mmaped_file_rs/src/file_entry.rs

@@ -1,10 +1,12 @@
-use magnus::{StaticSymbol, Symbol};
+use magnus::Symbol;
+use serde::Deserialize;
+use serde_json::value::RawValue;
+use smallvec::SmallVec;
 use std::fmt::Write;
 use std::str;
 
 use crate::error::{MmapError, RubyError};
 use crate::file_info::FileInfo;
-use crate::parser::{self, MetricText};
 use crate::raw_entry::RawEntry;
 use crate::Result;
 use crate::{SYM_GAUGE, SYM_LIVESUM, SYM_MAX, SYM_MIN};
@@ -16,6 +18,16 @@ pub struct FileEntry {
     pub meta: EntryMetadata,
 }
 
+/// String slices pointing to the fields of a borrowed `Entry`'s JSON data.
+#[derive(Deserialize, Debug)]
+pub struct MetricText<'a> {
+    pub family_name: &'a str,
+    pub metric_name: &'a str,
+    pub labels: SmallVec<[&'a str; 4]>,
+    #[serde(borrow)]
+    pub values: SmallVec<[&'a RawValue; 4]>,
+}
+
 /// The primary data payload for a `FileEntry`, the JSON string and the
 /// associated pid, if significant. Used as the key for `EntryMap`.
 #[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Debug)]
@@ -80,7 +92,7 @@ impl<'a> BorrowedData<'a> {
 #[derive(Clone, Debug)]
 pub struct EntryMetadata {
     pub multiprocess_mode: Symbol,
-    pub type_: StaticSymbol,
+    pub type_: Symbol,
     pub value: f64,
 }
 
@@ -134,11 +146,16 @@ impl FileEntry {
         let mut processed_count = 0;
 
         for entry in entries {
-            let metrics_data = match parser::parse_metrics(&entry.data.json) {
-                Some(m) => m,
+            let metrics_data = match serde_json::from_str::<MetricText>(&entry.data.json) {
+                Ok(m) => {
+                    if m.labels.len() != m.values.len() {
+                        continue;
+                    }
+                    m
+                }
                 // We don't exit the function here so the total number of invalid
                 // entries can be calculated below.
-                None => continue,
+                Err(_) => continue,
             };
 
             match prev_name.as_ref() {
@@ -176,7 +193,7 @@ impl FileEntry {
         out.push_str(family_name);
         out.push(' ');
 
-        out.push_str(self.meta.type_.name().expect("name was invalid UTF-8"));
+        out.push_str(&self.meta.type_.name().expect("name was invalid UTF-8"));
         out.push('\n');
     }
 
@@ -197,16 +214,20 @@ impl FileEntry {
 
         let it = json_data.labels.iter().zip(json_data.values.iter());
 
-        for (i, (&key, &val)) in it.enumerate() {
+        for (i, (&key, val)) in it.enumerate() {
             out.push_str(key);
-
-            out.push_str("=\"");
-
-            // `null` values will be output as `""`.
-            if val != "null" {
-                out.push_str(val);
+            out.push('=');
+
+            match val.get() {
+                "null" => out.push_str("\"\""),
+                s if s.starts_with('"') => out.push_str(s),
+                s => {
+                    // Quote numeric values.
+                    out.push('"');
+                    out.push_str(s);
+                    out.push('"');
+                }
             }
-            out.push('"');
 
             if i < json_data.labels.len() - 1 {
                 out.push(',');
@@ -276,6 +297,20 @@ mod test {
                     "##}),
                 expected_err: None,
             },
+            TestCase {
+                name: "many labels",
+                multiprocess_mode: "min",
+                json: &[
+                    r#"["family","name",["label_a","label_b","label_c","label_d","label_e"],["value_a","value_b","value_c","value_d","value_e"]]"#,
+                ],
+                values: &[1.0],
+                pids: &["worker-1"],
+                expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
+                    # TYPE family gauge
+                    name{label_a="value_a",label_b="value_b",label_c="value_c",label_d="value_d",label_e="value_e"} 1
+                    "##}),
+                expected_err: None,
+            },
             TestCase {
                 name: "floating point shown",
                 multiprocess_mode: "min",
@@ -288,6 +323,44 @@ mod test {
                     "##}),
                 expected_err: None,
             },
+            TestCase {
+                name: "numeric value",
+                multiprocess_mode: "min",
+                json: &[
+                    r#"["family","name",["label_a","label_b","label_c"],["value_a",403,-0.2E5]]"#,
+                ],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
+                    # TYPE family gauge
+                    name{label_a="value_a",label_b="403",label_c="-0.2E5"} 1.5
+                    "##}),
+                expected_err: None,
+            },
+            TestCase {
+                name: "null value",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a","label_b"],["value_a",null]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
+                    # TYPE family gauge
+                    name{label_a="value_a",label_b=""} 1.5
+                    "##}),
+                expected_err: None,
+            },
+            TestCase {
+                name: "comma in value",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a","label_b"],["value_a","value,_b"]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
+                    # TYPE family gauge
+                    name{label_a="value_a",label_b="value,_b"} 1.5
+                    "##}),
+                expected_err: None,
+            },
             TestCase {
                 name: "no labels, pid significant",
                 multiprocess_mode: "all",
@@ -413,6 +486,138 @@ mod test {
                     RubyError::Runtime,
                 )),
             },
+            TestCase {
+                name: "too many values",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a"],["value_a","value,_b"]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "no values",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a"]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "no labels or values",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name","foo"]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "too many leading brackets",
+                multiprocess_mode: "min",
+                json: &[r#"[["family","name",["label_a","label_b"],["value_a","value_b"]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "too many trailing brackets",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a","label_b"],["value_a","value_b"]]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "too many leading label brackets",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",[["label_a","label_b"],["value_a","value_b"]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "too many leading label brackets",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",[["label_a","label_b"],["value_a","value_b"]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "too many leading value brackets",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a","label_b"],[["value_a","value_b"]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "misplaced bracket",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a","label_b"],]["value_a","value_b"]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "comma in numeric",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a","label_b"],["value_a",403,0]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
+            TestCase {
+                name: "non-e letter in numeric",
+                multiprocess_mode: "min",
+                json: &[r#"["family","name",["label_a","label_b"],["value_a",-2.0c5]]"#],
+                values: &[1.5],
+                pids: &["worker-1"],
+                expected_out: None,
+                expected_err: Some(MmapError::legacy(
+                    "Processed entries 0 != map entries 1".to_owned(),
+                    RubyError::Runtime,
+                )),
+            },
         ];
 
         for case in tc {
@@ -438,7 +643,7 @@ mod test {
                     path,
                     len: case.json.len(),
                     multiprocess_mode: Symbol::new(case.multiprocess_mode),
-                    type_: StaticSymbol::new("gauge"),
+                    type_: Symbol::new("gauge"),
                     pid: pid.to_string(),
                 };
                 file_infos.push(info);
@@ -544,7 +749,7 @@ mod test {
                 path,
                 len: json.len(),
                 multiprocess_mode: Symbol::new(case.multiprocess_mode),
-                type_: StaticSymbol::new(case.metric_type),
+                type_: Symbol::new(case.metric_type),
                 pid: "worker-1".to_string(),
             };
 

data/ext/fast_mmaped_file_rs/src/file_info.rs

@@ -1,5 +1,5 @@
 use magnus::exception::*;
-use magnus::{Error, RString, StaticSymbol, Symbol, Value};
+use magnus::{Error, RString, Symbol, Value};
 use std::ffi::OsString;
 use std::fs::File;
 use std::io::{self, Read, Seek};
@@ -18,7 +18,7 @@ pub struct FileInfo {
     pub path: PathBuf,
     pub len: usize,
     pub multiprocess_mode: Symbol,
-    pub type_: StaticSymbol,
+    pub type_: Symbol,
     pub pid: String,
 }
 
@@ -59,7 +59,7 @@ impl FileInfo {
         let multiprocess_mode = Symbol::from_value(params[1])
             .ok_or_else(|| err!(arg_error(), "expected multiprocess_mode to be a symbol"))?;
 
-        let type_ = StaticSymbol::from_value(params[2])
+        let type_ = Symbol::from_value(params[2])
             .ok_or_else(|| err!(arg_error(), "expected file type to be a symbol"))?;
 
         let pid = RString::from_value(params[3])
@@ -91,6 +91,11 @@ impl FileInfo {
 
         match self.file.read_to_end(buf) {
             Ok(n) if n == self.len => Ok(()),
+            // A worker may expand the file between our `stat` and `read`, no harm done.
+            Ok(n) if n > self.len => {
+                self.len = n;
+                Ok(())
+            }
             Ok(_) => Err(MmapError::io(
                 "read",
                 &self.path,
@@ -103,9 +108,11 @@ impl FileInfo {
 
 #[cfg(test)]
 mod test {
-    use magnus::{eval, RArray, StaticSymbol, Symbol};
+    use magnus::{eval, RArray, Symbol};
     use rand::{thread_rng, Rng};
     use sha2::{Digest, Sha256};
+    use std::fs;
+    use std::io::Write;
 
     use super::*;
     use crate::testhelper::TestFile;
@@ -165,10 +172,10 @@ mod test {
 
         let mut info = FileInfo {
             file,
-            path,
+            path: path.clone(),
             len: buf.len(),
             multiprocess_mode: Symbol::new("puma"),
-            type_: StaticSymbol::new("max"),
+            type_: Symbol::new("max"),
             pid: "worker-0_0".to_string(),
         };
 
@@ -187,4 +194,47 @@ mod test {
 
         assert_eq!(in_hash, out_hash, "content hashes");
     }
+
+    #[test]
+    fn test_read_from_file_resized() {
+        let _cleanup = unsafe { magnus::embed::init() };
+        let ruby = magnus::Ruby::get().unwrap();
+        crate::init(&ruby).unwrap();
+
+        const BUF_LEN: usize = 1 << 14; // 16KiB
+
+        // Create a buffer with random data.
+        let mut buf = vec![0u8; BUF_LEN];
+        thread_rng().fill(buf.as_mut_slice());
+
+        let TestFile {
+            file,
+            path,
+            dir: _dir,
+        } = TestFile::new(&buf);
+
+        let mut info = FileInfo {
+            file,
+            path: path.clone(),
+            len: buf.len(),
+            multiprocess_mode: Symbol::new("puma"),
+            type_: Symbol::new("max"),
+            pid: "worker-0_0".to_string(),
+        };
+
+        let mut resized_file = fs::OpenOptions::new()
+            .write(true)
+            .append(true)
+            .open(path)
+            .unwrap();
+
+        // Write data to file after it has been `stat`ed in the
+        // constructor.
+        resized_file.write_all(&[1; 1024]).unwrap();
+
+        let mut out_buf = Vec::new();
+        info.read_from_file(&mut out_buf).unwrap();
+
+        assert_eq!(BUF_LEN + 1024, info.len, "resized file updated len");
+    }
 }

data/ext/fast_mmaped_file_rs/src/lib.rs

@@ -12,7 +12,6 @@ pub mod file_info;
 mod macros;
 pub mod map;
 pub mod mmap;
-pub mod parser;
 pub mod raw_entry;
 pub mod util;