vinted-prometheus-client-mmap 1.2.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38b385249d3ee5faca8bf41dcc9bc35ec625b4c6c11c2b90b612cb9fcf863bb1
-  data.tar.gz: 41f10da86eb0b8d5742d23c7670af5cfa2f85dbb9b57040e6b1183b5d7ffef0e
+  metadata.gz: 86a6083d5e7a88ac1e5e96a6660ebbd70fa31cecb1b1e6e324e26c4e91279713
+  data.tar.gz: a14011ec2a97f717bf0a18913fdde846b29ad742acb69b846b9f77c93ee7004f
 SHA512:
-  metadata.gz: 200a343904c8a993eb1495cd4128f307fd09a8e76f57b0d2af20301dda52ec5a9a0cca41300a90777b0e551bf2c9fd08eb18aacaa7a421c2b22543580e6ea2a9
-  data.tar.gz: 9f0e0b316c993cf1c5f531c6f9bfae540c287e32c044d43e5738a2753ab37e1ea230b9b69371daf4082b4046c957fc0e25c584012e04be5eae17885944697096
+  metadata.gz: e735c20e4bca0539a7e1219a1c9a0f7d3c61f838af04928e3d75d293bbf090be50ca9708aa0819234dc60025baa578795c65c8100da0550022cf4c0867186a65
+  data.tar.gz: bad6fe000c9a94c57797f3337fa961759a25660a337794d644b1edad8402b3a95314e51add28d6c9d3f4749ee09af3112658f13db8e95b2bb18fade77c580d6e

data/ext/fast_mmaped_file_rs/src/error.rs

@@ -135,7 +135,7 @@ impl MmapError {
         }
     }
 
-    pub fn out_of_bounds<T: Display>(index: T, len: T) -> Self {
+    pub fn out_of_bounds<T: Display>(index: T, len: T) -> Self {      
         MmapError::OutOfBounds {
             index: index.to_string(),
             len: len.to_string(),

data/ext/fast_mmaped_file_rs/src/exemplars.rs

@@ -1,4 +1,4 @@
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Exemplar {
     // Labels (set of label names/values). Only 1 for now.
     // Value -> f64.
@@ -8,18 +8,18 @@ pub struct Exemplar {
     // The combined length of the label names and values of an Exemplar's LabelSet MUST NOT exceed 128 UTF-8 character code points. 
     // 4 bytes max per code point.
     // So, we need to allocate 128*4 = 512 bytes for the label names and values.
-    LabelName: &str,
-    LabelValue: &str,
-    Value: f64,
-    Timestamp: u64,
+    pub label_name: String,
+    
+    pub label_value: String,
+    pub value: f64,
+    pub timestamp: u128,
 }
 
-pub struct Label {
-    Name: &str,
-    Value: &str,
-}
+use serde::{Deserialize, Serialize};
+
+use crate::size_of;
 
-pub const EXEMPLAR_ENTRY_MAX_SIZE_BYTES:u64 = 512 + size_of::<f64>() + size_of::<u64>();
+pub const EXEMPLAR_ENTRY_MAX_SIZE_BYTES:usize = 512 + size_of::<f64>() + size_of::<u64>();
 
 // Key -> use the old one.
 // Value -> allocate EXEMPLAR_ENTRY_MAX_SIZE_BYTES. If it exceeds this, we need to return an error. Use JSON.

data/ext/fast_mmaped_file_rs/src/file_entry.rs

@@ -7,6 +7,7 @@ use std::fmt::Write;
 use std::str;
 
 use crate::error::{MmapError, RubyError};
+use crate::exemplars::Exemplar;
 use crate::file_info::FileInfo;
 use crate::raw_entry::RawEntry;
 use crate::Result;
@@ -104,34 +105,72 @@ impl<'a> BorrowedData<'a> {
 pub struct EntryMetadata {
     pub multiprocess_mode: Symbol,
     pub type_: Symbol,
-    pub value: f64,
+    pub value: Option<f64>,
+    pub ex: Option<Exemplar>,
 }
 
 impl EntryMetadata {
     /// Construct a new `FileEntry`, copying the JSON string from the `RawEntry`
     /// into an internal buffer.
     pub fn new(mmap_entry: &RawEntry, file: &FileInfo) -> Result<Self> {
+        if file.type_.to_string() == "exemplar" {
+            let ex = mmap_entry.exemplar();
+
+            
+
+            return Ok(EntryMetadata {
+                multiprocess_mode: file.multiprocess_mode,
+                type_: file.type_,
+                value: None,
+                ex: Some(ex),
+            })
+        }
+
         let value = mmap_entry.value();
 
         Ok(EntryMetadata {
             multiprocess_mode: file.multiprocess_mode,
             type_: file.type_,
-            value,
+            value: Some(value),
+            ex: None,
         })
     }
 
     /// Combine values with another `EntryMetadata`.
     pub fn merge(&mut self, other: &Self) {
-        if self.type_ == SYM_GAUGE {
-            match self.multiprocess_mode {
-                s if s == SYM_MIN => self.value = self.value.min(other.value),
-                s if s == SYM_MAX => self.value = self.value.max(other.value),
-                s if s == SYM_LIVESUM => self.value += other.value,
-                _ => self.value = other.value,
+        if other.ex.is_some() {
+            let otherex = other.ex.clone().unwrap();
+            
+            if self.ex.is_some() {
+                let selfex = self.ex.clone().unwrap();
+
+                if selfex.timestamp < otherex.timestamp {
+                    self.ex = other.ex.clone();
+                }
+            } else {
+                self.ex = other.ex.clone();
             }
-        } else {
-            self.value += other.value;
         }
+        if other.value.is_some() {
+            if self.value.is_none() {
+                self.value = other.value;
+            } else {
+                let other_value = other.value.unwrap();
+                let self_value = self.value.unwrap();
+
+                if self.type_ == SYM_GAUGE {
+                    match self.multiprocess_mode {
+                        s if s == SYM_MIN => self.value = Some(self_value.min(other_value)),
+                        s if s == SYM_MAX => self.value = Some(self_value.max(other_value)),
+                        s if s == SYM_LIVESUM => self.value = Some(self_value + other_value),
+                        _ => self.value = Some(other_value),
+                    }
+                } else {
+                    self.value = Some(self_value + other_value);
+                }
+            }
+        }
+        
     }
 
     /// Validate if pid is significant for metric.
@@ -149,9 +188,26 @@ use std::collections::hash_map::DefaultHasher;
 use std::collections::HashMap;
 use std::hash::Hash;
 use std::hash::Hasher;
-use std::time::{SystemTime, UNIX_EPOCH};
 
 use std::io::Write as OtherWrite;
+
+fn exemplar_to_proto(e: &Exemplar) -> io::prometheus::client::Exemplar {
+    let seconds = e.timestamp / (1000 * 1000 * 1000);
+    let nanos = e.timestamp % (1000 * 1000 * 1000);
+
+    io::prometheus::client::Exemplar {
+        label: vec![io::prometheus::client::LabelPair {
+            name: Some(e.label_name.clone()),
+            value: Some(e.label_value.clone()),
+        }],
+        value: Some(e.value),
+        timestamp: Some(prost_types::Timestamp {
+            seconds: seconds as i64,
+            nanos: nanos as i32,
+        }),
+    }
+}
+
 impl FileEntry {
     pub fn trim_quotes(s: &str) -> String {
         let mut chars = s.chars();
@@ -229,26 +285,17 @@ impl FileEntry {
                             // Get the final u64 hash value
                             let hash_value = hasher.finish();
 
-                            let start = SystemTime::now();
-                            let since_the_epoch = start
-                                .duration_since(UNIX_EPOCH)
-                                .expect("Time went backwards");
-
                             m.counter = Some(io::prometheus::client::Counter {
-                                value: Some(gr.0.meta.value),
+                                value: gr.0.meta.value,
                                 created_timestamp: None,
-                                exemplar: Some(io::prometheus::client::Exemplar{
-                                    label: vec![
-                                        io::prometheus::client::LabelPair {
-                                            name: Some("traceID".to_string()),
-                                            value: Some("123456789".to_string()),
-                                        }
-                                    ],
-                                    value: Some(gr.0.meta.value),
-                                    timestamp: Some(prost_types::Timestamp { seconds:since_the_epoch.as_secs() as i64 , nanos: since_the_epoch.as_nanos() as i32 }),
-                                }),
+                                exemplar: None,
                             });
 
+                            if gr.0.meta.ex.is_some() {
+                                m.counter.as_mut().unwrap().exemplar =
+                                    Some(exemplar_to_proto(gr.0.meta.ex.as_ref().unwrap()));
+                            }
+
                             mtrcs.insert(hash_value, m);
                             metric_types.insert(hash_value, "counter");
                             metric_names.insert(hash_value, gr.1.metric_name);
@@ -266,7 +313,7 @@ impl FileEntry {
                             let hash_value = hasher.finish();
 
                             m.gauge = Some(io::prometheus::client::Gauge {
-                                value: Some(gr.0.meta.value),
+                                value: gr.0.meta.value,
                             });
                             mtrcs.insert(hash_value, m);
                             metric_types.insert(hash_value, "gauge");
@@ -312,9 +359,14 @@ impl FileEntry {
 
                                         let mut curf: f64 =
                                             bucket.cumulative_count_float.unwrap_or_default();
-                                        curf += gr.0.meta.value;
+                                        curf += gr.0.meta.value.unwrap();
 
                                         bucket.cumulative_count_float = Some(curf);
+
+                                        if gr.0.meta.ex.is_some() {
+                                            bucket.exemplar =
+                                                Some(exemplar_to_proto(gr.0.meta.ex.as_ref().unwrap()));
+                                        }
                                     }
                                 }
                                 None => {
@@ -333,9 +385,9 @@ impl FileEntry {
                                         final_metric_name = stripped;
                                     }
 
-                                    let buckets = vec![io::prometheus::client::Bucket {
+                                    let mut buckets = vec![io::prometheus::client::Bucket {
                                         cumulative_count: None,
-                                        cumulative_count_float: Some(gr.0.meta.value),
+                                        cumulative_count_float: gr.0.meta.value,
                                         upper_bound: Some(
                                             le.expect(
                                                 &format!("got no LE for {}", gr.1.metric_name)
@@ -344,6 +396,11 @@ impl FileEntry {
                                         ),
                                         exemplar: None,
                                     }];
+
+                                    if gr.0.meta.ex.is_some() {
+                                        buckets[0].exemplar =
+                                            Some(exemplar_to_proto(gr.0.meta.ex.as_ref().unwrap()));
+                                    }
                                     m.label = m
                                         .label
                                         .into_iter()
@@ -413,10 +470,10 @@ impl FileEntry {
                                     if gr.1.metric_name.ends_with("_count") {
                                         let samplecount = smry.sample_count.unwrap_or_default();
                                         smry.sample_count =
-                                            Some((gr.0.meta.value as u64) + samplecount);
+                                            Some((gr.0.meta.value.unwrap() as u64) + samplecount);
                                     } else if gr.1.metric_name.ends_with("_sum") {
                                         let samplesum: f64 = smry.sample_sum.unwrap_or_default();
-                                        smry.sample_sum = Some(gr.0.meta.value + samplesum);
+                                        smry.sample_sum = Some(gr.0.meta.value.unwrap() + samplesum);
                                     } else {
                                         let mut found_quantile = false;
                                         for qntl in &mut smry.quantile {
@@ -425,7 +482,7 @@ impl FileEntry {
                                             }
 
                                             let mut curq: f64 = qntl.quantile.unwrap_or_default();
-                                            curq += gr.0.meta.value;
+                                            curq += gr.0.meta.value.unwrap();
 
                                             qntl.quantile = Some(curq);
                                             found_quantile = true;
@@ -434,7 +491,7 @@ impl FileEntry {
                                         if !found_quantile {
                                             smry.quantile.push(io::prometheus::client::Quantile {
                                                 quantile: quantile,
-                                                value: Some(gr.0.meta.value),
+                                                value: gr.0.meta.value,
                                             });
                                         }
                                     }
@@ -455,7 +512,7 @@ impl FileEntry {
                                             gr.1.metric_name.strip_suffix("_count").unwrap();
                                         m.summary = Some(io::prometheus::client::Summary {
                                             quantile: vec![],
-                                            sample_count: Some(gr.0.meta.value as u64),
+                                            sample_count: Some(gr.0.meta.value.unwrap() as u64),
                                             sample_sum: None,
                                             created_timestamp: None,
                                         });
@@ -464,14 +521,14 @@ impl FileEntry {
                                             gr.1.metric_name.strip_suffix("_sum").unwrap();
                                         m.summary = Some(io::prometheus::client::Summary {
                                             quantile: vec![],
-                                            sample_sum: Some(gr.0.meta.value),
+                                            sample_sum: Some(gr.0.meta.value.unwrap()),
                                             sample_count: None,
                                             created_timestamp: None,
                                         });
                                     } else {
                                         let quantiles = vec![io::prometheus::client::Quantile {
                                             quantile: quantile,
-                                            value: Some(gr.0.meta.value),
+                                            value: gr.0.meta.value,
                                         }];
                                         m.summary = Some(io::prometheus::client::Summary {
                                             quantile: quantiles,
@@ -487,6 +544,9 @@ impl FileEntry {
                                 }
                             }
                         }
+                        "exemplar" => {
+                            // Exemplars are handled later on.
+                        }
                         mtype => {
                             panic!("unhandled metric type {}", mtype)
                         }
@@ -532,6 +592,8 @@ impl FileEntry {
         unsafe { Ok(str::from_utf8_unchecked(buffer.get_ref()).to_string()) }
     }
 
+    
+
     /// Convert the sorted entries into a String in Prometheus metrics format.
     pub fn entries_to_string(entries: Vec<FileEntry>) -> Result<String> {
         // We guesstimate that lines are ~100 bytes long, preallocate the string to
@@ -568,7 +630,7 @@ impl FileEntry {
 
             entry.append_entry(metrics_data, &mut out)?;
 
-            writeln!(&mut out, " {}", entry.meta.value)
+            writeln!(&mut out, " {}", entry.meta.value.unwrap())
                 .map_err(|e| MmapError::Other(format!("Failed to append to output: {e}")))?;
 
             processed_count += 1;
@@ -1182,7 +1244,7 @@ mod test {
             entry_a.meta.merge(&entry_b.meta);
 
             assert_eq!(
-                case.expected_value, entry_a.meta.value,
+                case.expected_value, entry_a.meta.value.unwrap(),
                 "test case: {name} - value"
             );
         }

data/ext/fast_mmaped_file_rs/src/lib.rs

@@ -14,6 +14,7 @@ pub mod map;
 pub mod mmap;
 pub mod raw_entry;
 pub mod util;
+pub mod exemplars;
 
 pub mod io {
     pub mod prometheus {
@@ -82,6 +83,7 @@ fn init(ruby: &Ruby) -> magnus::error::Result<()> {
     klass.define_method("used=", method!(MmapedFile::save_used, 1))?;
     klass.define_method("fetch_entry", method!(MmapedFile::fetch_entry, 3))?;
     klass.define_method("upsert_entry", method!(MmapedFile::upsert_entry, 3))?;
+    klass.define_method("upsert_exemplar", method!(MmapedFile::upsert_exemplar, 5))?;
 
     Ok(())
 }

data/ext/fast_mmaped_file_rs/src/map.rs

@@ -142,22 +142,39 @@ impl EntryMap {
         let mut pos = HEADER_SIZE;
 
         while pos + size_of::<u32>() < used {
-            let raw_entry = RawEntry::from_slice(&source[pos..used])?;
-
-            if pos + raw_entry.total_len() > used {
-                return Err(MmapError::PromParsing(format!(
-                    "source file {} corrupted, used {used} < stored data length {}",
-                    file_info.path.display(),
-                    pos + raw_entry.total_len()
-                )));
-            }
+            let raw_entry: RawEntry;
+
+            if file_info.type_.to_string() == "exemplar" {
+                raw_entry = RawEntry::from_slice_exemplar(&source[pos..used])?;
+
+                if pos + raw_entry.total_len_exemplar() > used {
+                    return Err(MmapError::PromParsing(format!(
+                        "source file {} corrupted, used {used} < stored data length {}",
+                        file_info.path.display(),
+                        pos + raw_entry.total_len()
+                    )));
+                }
 
+                pos += raw_entry.total_len_exemplar();
+
+            } else {
+                raw_entry = RawEntry::from_slice(&source[pos..used])?;
+
+                if pos + raw_entry.total_len() > used {
+                    return Err(MmapError::PromParsing(format!(
+                        "source file {} corrupted, used {used} < stored data length {}",
+                        file_info.path.display(),
+                        pos + raw_entry.total_len()
+                    )));
+                }
+
+                pos += raw_entry.total_len();
+            }
+            
             let meta = EntryMetadata::new(&raw_entry, &file_info)?;
             let data = BorrowedData::new(&raw_entry, &file_info, meta.is_pid_significant())?;
 
             self.merge_or_store(data, meta)?;
-
-            pos += raw_entry.total_len();
         }
 
         Ok(())
@@ -198,7 +215,8 @@ mod test {
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("max"),
                     type_: Symbol::new("gauge"),
-                    value: 1.0,
+                    value: Some(1.0),
+                    ex: None,
                 },
             },
             FileEntry {
@@ -209,7 +227,8 @@ mod test {
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("max"),
                     type_: Symbol::new("gauge"),
-                    value: 1.0,
+                    value: Some(1.0),
+                    ex: None,
                 },
             },
             FileEntry {
@@ -220,7 +239,8 @@ mod test {
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("max"),
                     type_: Symbol::new("gauge"),
-                    value: 1.0,
+                    value: Some(1.0),
+                    ex: None,
                 },
             },
             FileEntry {
@@ -231,7 +251,8 @@ mod test {
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("max"),
                     type_: Symbol::new("gauge"),
-                    value: 1.0,
+                    value: Some(1.0),
+                    ex: None,
                 },
             },
             FileEntry {
@@ -242,7 +263,8 @@ mod test {
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("all"),
                     type_: Symbol::new("gauge"),
-                    value: 1.0,
+                    value: Some(1.0),
+                    ex: None,
                 },
             },
             FileEntry {
@@ -253,7 +275,8 @@ mod test {
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("all"),
                     type_: Symbol::new("gauge"),
-                    value: 1.0,
+                    value: Some(1.0),
+                    ex: None,
                 },
             },
         ];
@@ -294,7 +317,8 @@ mod test {
             meta: EntryMetadata {
                 multiprocess_mode: Symbol::new("all"),
                 type_: Symbol::new("gauge"),
-                value: 1.0,
+                value: Some(1.0),
+                ex: None,
             },
         };
 
@@ -306,7 +330,8 @@ mod test {
             meta: EntryMetadata {
                 multiprocess_mode: Symbol::new("all"),
                 type_: Symbol::new("gauge"),
-                value: 5.0,
+                value: Some(5.0),
+                ex: None,
             },
         };
 
@@ -318,7 +343,8 @@ mod test {
             meta: EntryMetadata {
                 multiprocess_mode: Symbol::new("all"),
                 type_: Symbol::new("gauge"),
-                value: 100.0,
+                value: Some(100.0),
+                ex: None,
             },
         };
 
@@ -330,7 +356,8 @@ mod test {
             meta: EntryMetadata {
                 multiprocess_mode: Symbol::new("all"),
                 type_: Symbol::new("gauge"),
-                value: 1.0,
+                value: Some(100.0),
+                ex: None,
             },
         };
 
@@ -345,7 +372,7 @@ mod test {
 
         assert_eq!(
             5.0,
-            map.0.get(&starting_entry.data).unwrap().value,
+            map.0.get(&starting_entry.data).unwrap().value.unwrap(),
             "value updated"
         );
         assert_eq!(1, map.0.len(), "no entry added");
@@ -359,7 +386,7 @@ mod test {
 
         assert_eq!(
             5.0,
-            map.0.get(&starting_entry.data).unwrap().value,
+            map.0.get(&starting_entry.data).unwrap().value.unwrap(),
             "value unchanged"
         );
 
@@ -371,7 +398,7 @@ mod test {
 
         assert_eq!(
             5.0,
-            map.0.get(&starting_entry.data).unwrap().value,
+            map.0.get(&starting_entry.data).unwrap().value.unwrap(),
             "value unchanged"
         );
         assert_eq!(3, map.0.len(), "entry added");

data/ext/fast_mmaped_file_rs/src/mmap/inner.rs

@@ -9,10 +9,13 @@ use std::path::PathBuf;
 
 use crate::error::{MmapError, RubyError};
 use crate::raw_entry::RawEntry;
-use crate::util::CheckedOps;
+use crate::exemplars::{Exemplar, EXEMPLAR_ENTRY_MAX_SIZE_BYTES};
+
+use crate::util::{read_exemplar, CheckedOps};
 use crate::util::{self, errno, read_f64, read_u32};
 use crate::Result;
 use crate::HEADER_SIZE;
+use std::iter;
 
 /// A mmapped file and its metadata. Ruby never directly interfaces
 /// with this struct.
@@ -139,6 +142,63 @@ impl InnerMmap {
         Ok(position)
     }
 
+    pub unsafe fn initialize_entry_exemplar(&mut self, key: &[u8], ex: Exemplar) -> Result<usize> {
+        // CAST: no-op on 32-bit, widening on 64-bit.
+        let current_used = self.load_used()? as usize;
+        let entry_length = RawEntry::calc_total_len_exemplar(key.len())?;
+
+        let new_used = current_used.add_chk(entry_length)?;
+
+        // Increasing capacity requires expanding the file and re-mmapping it, we can't
+        // perform this from `InnerMmap`.
+        if self.capacity() < new_used {
+            return Err(MmapError::Other(format!(
+                "mmap capacity {} less than {}",
+                self.capacity(),
+                new_used
+            )));
+        }
+
+        let bytes = self.map.as_mut();
+        let value_offset = RawEntry::save_exemplar(&mut bytes[current_used..new_used], key, ex)?;
+
+        // Won't overflow as value_offset is less than new_used.
+        let position = current_used + value_offset;
+        let new_used32 = util::cast_chk::<_, u32>(new_used, "used")?;
+
+        self.save_used(new_used32)?;
+        Ok(position)
+    }
+
+    pub fn save_exemplar(&mut self, offset: usize, exemplar: Exemplar) -> Result<()> {
+        if self.len.add_chk(size_of::<Exemplar>())? <= offset {
+            return Err(MmapError::out_of_bounds(
+                offset + size_of::<f64>(),
+                self.len,
+            ));
+        }
+
+        if offset < HEADER_SIZE {
+            return Err(MmapError::Other(format!(
+                "writing to offset {offset} would overwrite file header"
+            )));
+        }
+
+        let val = serde_json::to_string(&exemplar).unwrap();
+
+        let value_bytes = val.as_bytes();
+
+        let mut value_bytes = value_bytes.to_vec();
+        value_bytes.extend(iter::repeat(0).take(EXEMPLAR_ENTRY_MAX_SIZE_BYTES - value_bytes.len()));
+
+        let value_range = self.item_range(offset, EXEMPLAR_ENTRY_MAX_SIZE_BYTES)?;
+
+        let bytes = self.map.as_mut();
+        bytes[value_range].copy_from_slice(&value_bytes);
+
+        Ok(())
+    }
+
     /// Save a metrics value to an existing entry in the mmap.
     pub fn save_value(&mut self, offset: usize, value: f64) -> Result<()> {
         if self.len.add_chk(size_of::<f64>())? <= offset {
@@ -174,6 +234,17 @@ impl InnerMmap {
         read_f64(self.map.as_ref(), offset)
     }
 
+    pub fn load_exemplar(&mut self, offset: usize) -> Result<Exemplar> {
+        if self.len.add_chk(EXEMPLAR_ENTRY_MAX_SIZE_BYTES)? <= offset {
+            return Err(MmapError::out_of_bounds(
+                offset + EXEMPLAR_ENTRY_MAX_SIZE_BYTES,
+                self.len,
+            ));
+        }
+
+        read_exemplar(self.map.as_mut(), offset)
+    }
+
     /// The length of data written to the file.
     /// With a new file this is only set when Ruby calls `slice` on
     /// `FastMmapedFileRs`, so even if data has been written to the

data/ext/fast_mmaped_file_rs/src/mmap.rs

@@ -15,6 +15,7 @@ use std::sync::RwLock;
 
 use crate::err;
 use crate::error::MmapError;
+use crate::exemplars::Exemplar;
 use crate::file_entry::FileEntry;
 use crate::map::EntryMap;
 use crate::raw_entry::RawEntry;
@@ -76,6 +77,8 @@ const STR_SHARED: c_ulong = 1 << (14);
 #[magnus::wrap(class = "FastMmapedFileRs", free_immediately, size)]
 pub struct MmapedFile(RwLock<Option<InnerMmap>>);
 
+use std::time::{SystemTime, UNIX_EPOCH};
+
 impl MmapedFile {
     /// call-seq:
     ///   new(file)
@@ -320,6 +323,61 @@ impl MmapedFile {
         rs_self.load_value(value_offset)
     }
 
+    pub fn upsert_exemplar(
+        rb_self: Obj<Self>,
+        positions: RHash,
+        key: RString,
+        value: f64,
+        exemplar_name: RString,
+        exemplar_value: RString,
+    ) -> magnus::error::Result<f64> {
+        let rs_self = &*rb_self;
+        let position: Option<Fixnum> = positions.lookup(key)?;
+
+        let start = SystemTime::now();
+        let since_the_epoch = start
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards");
+
+        let ex: Exemplar = Exemplar {
+            label_name: unsafe { exemplar_name.as_str().unwrap().into() },
+            label_value: unsafe { exemplar_value.as_str().unwrap().into() },
+            value: value,
+            timestamp: since_the_epoch.as_nanos(),
+        };
+
+        if let Some(pos) = position {
+            let pos = pos.to_usize()?;
+            return rs_self
+                .inner_mut(|inner| {
+                    inner.save_exemplar(pos, ex)?;
+
+                    // TODO just return `value` here instead of loading it?
+                    // This is how the C implementation did it, but I don't
+                    // see what the extra load gains us.
+                    let ex = inner.load_exemplar(pos);
+
+                    Ok(ex.unwrap().value)
+                })
+                .map_err(|e| e.into());
+        }
+
+
+        rs_self.check_expand_exemplar(rb_self, key.len())?;
+
+        let value_offset: usize = rs_self.inner_mut(|inner| {
+            // SAFETY: We must not call any Ruby code for the lifetime of this borrow.
+            unsafe { inner.initialize_entry_exemplar(key.as_slice(), ex) }
+        })?;
+
+        // CAST: no-op on 64-bit, widening on 32-bit.
+        positions.aset(key, Integer::from_u64(value_offset as u64))?;
+
+        let ex = rs_self.load_exemplar(value_offset);
+
+        Ok(ex.unwrap().value)
+    }
+
     /// Update the value of an existing entry, if present. Otherwise create a new entry
     /// for the key.
     pub fn upsert_entry(
@@ -469,6 +527,23 @@ impl MmapedFile {
         Ok(())
     }
 
+    /// Check that the mmap is large enough to contain the value to be added,
+    /// and expand it to fit if necessary.
+    fn check_expand_exemplar(&self, rb_self: Obj<Self>, key_len: usize) -> magnus::error::Result<()> {
+        // CAST: no-op on 32-bit, widening on 64-bit.
+        let used = self.inner(|inner| inner.load_used())? as usize;
+        let entry_len = RawEntry::calc_total_len_exemplar(key_len)?;
+
+        // We need the mmapped region to contain at least one byte beyond the
+        // written data to create a NUL- terminated C string. Validate that
+        // new length does not exactly match or exceed the length of the mmap.
+        while self.capacity() <= used.add_chk(entry_len)? {
+            self.expand_to_fit(rb_self, self.capacity().mul_chk(2)?)?;
+        }
+
+        Ok(())
+    }
+
     /// Expand the underlying file until it is long enough to fit `target_cap`.
     /// This will remove the existing mmap, expand the file, then update any
     /// strings held by the `WeakMap` to point to the newly mmapped address.
@@ -555,6 +630,11 @@ impl MmapedFile {
             .map_err(|e| e.into())
     }
 
+    fn load_exemplar<'a, 'b>(&'a self, position: usize) -> magnus::error::Result<Exemplar> {
+        self.inner_mut(|inner| inner.load_exemplar(position))
+            .map_err(|e| e.into())
+    }
+
     fn as_mut_ptr(&self) -> *mut c_char {
         // UNWRAP: This is actually infallible, but we need to
         // wrap it in a `Result` for use with `inner()`.
@@ -655,13 +735,14 @@ impl MmapedFile {
 
 #[cfg(test)]
 mod test {
+    use super::*;
+    use core::panic;
     use magnus::error::Error;
     use magnus::eval;
     use magnus::Range;
     use nix::unistd::{sysconf, SysconfVar};
     use std::mem::size_of;
 
-    use super::*;
     use crate::raw_entry::RawEntry;
     use crate::testhelper::TestFile;
 

data/ext/fast_mmaped_file_rs/src/raw_entry.rs

@@ -1,6 +1,7 @@
 use std::mem::size_of;
 
 use crate::error::MmapError;
+use crate::exemplars::{Exemplar, EXEMPLAR_ENTRY_MAX_SIZE_BYTES};
 use crate::util;
 use crate::util::CheckedOps;
 use crate::Result;
@@ -13,6 +14,42 @@ pub struct RawEntry<'a> {
 }
 
 impl<'a> RawEntry<'a> {
+    pub fn save_exemplar(bytes: &'a mut [u8], key: &[u8], value: Exemplar) -> Result<usize> {
+        let total_len = Self::calc_total_len_exemplar(key.len())?;
+
+        if total_len > bytes.len() {
+            return Err(MmapError::Other(format!(
+                "entry length {total_len} larger than slice length {}",
+                bytes.len()
+            )));
+        }
+
+        let val = serde_json::to_string(&value).unwrap();
+
+        // CAST: `calc_len` runs `check_encoded_len`, we know the key len
+        // is less than i32::MAX. No risk of overflows or failed casts.
+        let key_len: u32 = key.len() as u32;
+
+        // Write the key length to the mmap.
+        bytes[..size_of::<u32>()].copy_from_slice(&key_len.to_ne_bytes());
+
+        // Advance slice past the size.
+        let bytes = &mut bytes[size_of::<u32>()..];
+
+        bytes[..key.len()].copy_from_slice(key);
+
+        // Advance to end of key.
+        let bytes = &mut bytes[key.len()..];
+
+        let pad_len = Self::padding_len(key.len());
+        bytes[..pad_len].fill(b' ');
+        let bytes = &mut bytes[pad_len..];
+
+        bytes[..val.len()].copy_from_slice(val.as_bytes());
+
+        Self::calc_value_offset(key.len())
+    }
+
     /// Save an entry to the mmap, returning the value offset in the newly created entry.
     pub fn save(bytes: &'a mut [u8], key: &[u8], value: f64) -> Result<usize> {
         let total_len = Self::calc_total_len(key.len())?;
@@ -66,6 +103,23 @@ impl<'a> RawEntry<'a> {
         Ok(Self { bytes, encoded_len })
     }
 
+    pub fn from_slice_exemplar(bytes: &'a [u8]) -> Result<Self> {
+        // CAST: no-op on 32-bit, widening on 64-bit.
+        let encoded_len = util::read_u32(bytes, 0)? as usize;
+
+        let total_len = Self::calc_total_len_exemplar(encoded_len)?;
+
+        // Confirm the value is in bounds of the slice provided.
+        if total_len > bytes.len() {
+            return Err(MmapError::out_of_bounds(total_len, bytes.len()));
+        }
+
+        // Advance slice past length int and cut at end of entry.
+        let bytes = &bytes[size_of::<u32>()..total_len];
+
+        Ok(Self { bytes, encoded_len })
+    }
+
     /// Read the `f64` value of an entry from memory.
     #[inline]
     pub fn value(&self) -> f64 {
@@ -77,6 +131,15 @@ impl<'a> RawEntry<'a> {
         util::read_f64(self.bytes, offset).unwrap()
     }
 
+    pub fn exemplar(&self) -> Exemplar {
+        // We've stripped off the leading u32, don't include that here.
+        let offset = self.encoded_len + Self::padding_len(self.encoded_len);
+
+        // UNWRAP: We confirm in the constructor that the value offset
+        // is in-range for the slice.
+        util::read_exemplar(self.bytes, offset).unwrap()
+    }
+
     /// The length of the entry key without padding.
     #[inline]
     pub fn encoded_len(&self) -> usize {
@@ -97,6 +160,12 @@ impl<'a> RawEntry<'a> {
         Self::calc_total_len(self.encoded_len).unwrap()
     }
 
+    #[inline]
+    pub fn total_len_exemplar(&self) -> usize {
+        // UNWRAP:: We confirmed in the constructor that this doesn't overflow.
+        Self::calc_total_len_exemplar(self.encoded_len).unwrap()
+    }
+
     /// Calculate the total length of an `MmapEntry`, including the string length,
     /// string, padding, and value. Validates encoding_len is within expected bounds.
     #[inline]
@@ -104,6 +173,11 @@ impl<'a> RawEntry<'a> {
         Self::calc_value_offset(encoded_len)?.add_chk(size_of::<f64>())
     }
 
+    #[inline]
+    pub fn calc_total_len_exemplar(encoded_len: usize) -> Result<usize> {
+        Self::calc_value_offset(encoded_len)?.add_chk(EXEMPLAR_ENTRY_MAX_SIZE_BYTES)
+    }
+
     /// Calculate the value offset of an `MmapEntry`, including the string length,
     /// string, padding. Validates encoding_len is within expected bounds.
     #[inline]

data/ext/fast_mmaped_file_rs/src/util.rs

@@ -5,6 +5,7 @@ use std::io;
 use std::mem::size_of;
 
 use crate::error::MmapError;
+use crate::exemplars::{Exemplar, EXEMPLAR_ENTRY_MAX_SIZE_BYTES};
 use crate::Result;
 
 /// Wrapper around `checked_add()` that converts failures
@@ -88,6 +89,24 @@ pub fn read_f64(buf: &[u8], offset: usize) -> Result<f64> {
     ))
 }
 
+pub fn read_exemplar(buf: &[u8], offset: usize) -> Result<Exemplar> {
+    if let Some(slice) = buf.get(offset..offset + EXEMPLAR_ENTRY_MAX_SIZE_BYTES) {
+        // UNWRAP: We can safely unwrap the conversion from slice to array as we
+       // can be sure the target array has same length as the source slice.
+       let out: &[u8; EXEMPLAR_ENTRY_MAX_SIZE_BYTES] = slice.try_into().expect("failed to convert slice to array");
+
+       let res: Vec<u8> = out.iter().cloned().filter(|&x| x != 0).collect();
+
+       let v: Exemplar = serde_json::from_slice(&res).expect("failed to convert string to Exemplar");
+        
+       return Ok(v)
+    }
+    Err(MmapError::out_of_bounds(
+        offset + EXEMPLAR_ENTRY_MAX_SIZE_BYTES,
+        buf.len(),
+    ))
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

data/lib/prometheus/client/counter.rb

@@ -10,11 +10,11 @@ module Prometheus
         :counter
       end
 
-      def increment(labels = {}, by = 1)
+      def increment(labels = {}, by = 1, exemplar_name = '', exemplar_value = '')
         raise ArgumentError, 'increment must be a non-negative number' if by < 0
 
         label_set = label_set_for(labels)
-        synchronize { @values[label_set].increment(by) }
+        synchronize { @values[label_set].increment(by, exemplar_name, exemplar_value) }
       end
 
       private

data/lib/prometheus/client/formats/protobuf.rb

@@ -29,7 +29,8 @@ module Prometheus
           end
 
           def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir, use_rust: true)
-            file_list = Dir.glob(File.join(path, '*.db')).sort
+            # NOTE(GiedriusS): need to ensure exemplar files go at the end because they add extra data.
+            file_list = Dir.glob(File.join(path, '*.db')).sort_by { |f| [f.include?('exemplar') ? 1 : 0, f] }
               .map {|f| Helper::PlainFile.new(f) }
               .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
 

data/lib/prometheus/client/histogram.rb

@@ -60,9 +60,9 @@ module Prometheus
         :histogram
       end
 
-      def observe(labels, value)
+      def observe(labels, value, exemplar_name = '', exemplar_value = '')
         label_set = label_set_for(labels)
-        synchronize { @values[label_set].observe(value) }
+        synchronize { @values[label_set].observe(value, exemplar_name, exemplar_value) }
       end
 
       private

data/lib/prometheus/client/mmaped_dict.rb

@@ -46,6 +46,10 @@ module Prometheus
         @m.upsert_entry(@positions, key, value)
       end
 
+      def write_exemplar(key, value, exemplar_id, exemplar_val)
+        @m.upsert_exemplar(@positions, key, value, exemplar_id, exemplar_val)
+      end
+
       def path
         @m.filepath if @m
       end

data/lib/prometheus/client/mmaped_value.rb

@@ -31,11 +31,7 @@ module Prometheus
           initialize_file if pid_changed?
 
           @value += amount
-          # TODO(GiedriusS): write exemplars too.
-          if @file_prefix != 'gauge'
-            puts "#{@name} exemplar name = #{exemplar_name}, exemplar_value = #{exemplar_value}"
-          end
-          write_value(@key, @value)
+          write_value(@key, @value, exemplar_name, exemplar_value)
           @value
         end
       end
@@ -120,12 +116,18 @@ module Prometheus
           unless @file.nil?
             @file.close
           end
+          unless @exemplar_file.nil?
+            @exemplar_file.close
+          end
           mmaped_file = Helper::MmapedFile.open_exclusive_file(@file_prefix)
+          exemplar_file = Helper::MmapedFile.open_exclusive_file('exemplar')
 
           @@files[@file_prefix] = MmapedDict.new(mmaped_file)
+          @@files['exemplar'] = MmapedDict.new(exemplar_file)
         end
 
         @file = @@files[@file_prefix]
+        @exemplar_file = @@files['exemplar']
         @key = rebuild_key
 
         @value = read_value(@key)
@@ -139,8 +141,12 @@ module Prometheus
         [@metric_name, @name, keys, values].to_json
       end
 
-      def write_value(key, val)
+      def write_value(key, val, exemplar_name = '', exemplar_value = '')
         @file.write_value(key, val)
+        # Exemplars are only defined on counters or histograms.
+        if @file_prefix == 'counter' or @file_prefix == 'histogram' and exemplar_name != '' and exemplar_value != ''
+          @exemplar_file.write_exemplar(key, val, exemplar_name, exemplar_value)
+        end
       rescue StandardError => e
         Prometheus::Client.logger.warn("writing value to #{@file.path} failed with #{e}")
         Prometheus::Client.logger.debug(e.backtrace.join("\n"))

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '1.2.2'.freeze
+    VERSION = '1.4.0'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vinted-prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.4.0
 platform: ruby
 authors:
 - Tobias Schmidt
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-26 00:00:00.000000000 Z
+date: 2024-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys