prometheus-client-mmap 0.23.1-x86_64-darwin → 0.24.4-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 259e3a6f9de504beed95cb99507637c294fd6abbdb5a299784525bfe932e3d1c
-  data.tar.gz: ee3ce3faee8d41074738e9f2d52ac5c70602bc893e2549a5b69ab98891ff7b23
+  metadata.gz: 11eecc761b0bd7cbcc8137e7097cb81595b0bbbe5b01058b1a601f01fd0405e9
+  data.tar.gz: c9d175042f2a64f0735d5a4b973506c56690a555e20bb115062c207e4389ba01
 SHA512:
-  metadata.gz: efa2b51434173c59706a0957bb54ceefce6b0d6db0323a5bb079e196e5a9f86e1828dcd89c53db514e274c89399a216ea90a03fd9af55d006bb13eded188727a
-  data.tar.gz: a0c2b029e3ee67ddd0795626260ba9eeff355239105b219206e14bbe0d38c511c63eb6987b442350b2d8fc7c9b4f7ce38c1d3bb8c7c9b02acb1e1ef6a882c9a0
+  metadata.gz: 3f5428ee9d38050a212df5d313c73ac4734ee6d0d174608ee5b72cd1d8ba586db82b41d30f356373b33a350ce9fa291d515b4e965dde309b83ca449dcd0d0b6b
+  data.tar.gz: e67748cf4f5d979b25068bf1d86fc828419301fc09425c40f54cb24aa0247dcf69aa9a295daca22b8458cd04597182603a62ae1b02e9195a88350c33c2efc885

data/README.md

@@ -200,6 +200,17 @@ Set `prometheus_multiproc_dir` environment variable to the path where you want m
 prometheus_multiproc_dir=/tmp
 ```
 
+### Multiprocess metrics via Rust extension
+
+If the environment variable `prometheus_rust_multiprocess_metrics=true` is set or if the `rust_multiprocess_metrics`
+configuration setting is `true` and the `fast_mmaped_file_rs` extension is available, it will be used to generate
+multiprocess metrics. This should be significantly faster than the C extension.
+
+### Read and write metrics via Rust extension
+
+If the environment variable `prometheus_rust_mmaped_file=true` is set then if the `fast_mmaped_file_rs`
+extension is available it will be used to read and write metrics from the mmapped file.
+
 ## Pitfalls
 
 ### PID cardinality

data/ext/fast_mmaped_file_rs/src/error.rs

@@ -9,7 +9,7 @@ use crate::util;
 use crate::PROM_EPARSING_ERROR;
 
 /// A lightweight representation of Ruby ExceptionClasses.
-#[derive(PartialEq, Clone, Copy, Debug)]
+#[derive(PartialEq, Eq, Clone, Copy, Debug)]
 pub enum RubyError {
     Arg,
     Encoding,
@@ -45,7 +45,7 @@ impl From<RubyError> for magnus::ExceptionClass {
 /// Errors returned internally within the crate. Methods called directly by Ruby return
 /// `magnus::error::Error` as do functions that interact heavily with Ruby. This can be
 /// converted into a `magnus::error::Error` at the boundary between Rust and Ruby.
-#[derive(PartialEq, Error, Debug)]
+#[derive(PartialEq, Eq, Error, Debug)]
 pub enum MmapError {
     /// A read or write was made while another thread had mutable access to the mmap.
     #[error("read/write operation attempted while mmap was being written to")]

data/ext/fast_mmaped_file_rs/src/file_entry.rs

@@ -1,4 +1,4 @@
-use magnus::{StaticSymbol, Symbol};
+use magnus::Symbol;
 use std::fmt::Write;
 use std::str;
 
@@ -80,7 +80,7 @@ impl<'a> BorrowedData<'a> {
 #[derive(Clone, Debug)]
 pub struct EntryMetadata {
     pub multiprocess_mode: Symbol,
-    pub type_: StaticSymbol,
+    pub type_: Symbol,
     pub value: f64,
 }
 
@@ -176,7 +176,7 @@ impl FileEntry {
         out.push_str(family_name);
         out.push(' ');
 
-        out.push_str(self.meta.type_.name().expect("name was invalid UTF-8"));
+        out.push_str(&self.meta.type_.name().expect("name was invalid UTF-8"));
         out.push('\n');
     }
 
@@ -438,7 +438,7 @@ mod test {
                     path,
                     len: case.json.len(),
                     multiprocess_mode: Symbol::new(case.multiprocess_mode),
-                    type_: StaticSymbol::new("gauge"),
+                    type_: Symbol::new("gauge"),
                     pid: pid.to_string(),
                 };
                 file_infos.push(info);
@@ -544,7 +544,7 @@ mod test {
                 path,
                 len: json.len(),
                 multiprocess_mode: Symbol::new(case.multiprocess_mode),
-                type_: StaticSymbol::new(case.metric_type),
+                type_: Symbol::new(case.metric_type),
                 pid: "worker-1".to_string(),
             };
 

data/ext/fast_mmaped_file_rs/src/file_info.rs

@@ -1,5 +1,5 @@
 use magnus::exception::*;
-use magnus::{Error, RString, StaticSymbol, Symbol, Value};
+use magnus::{Error, RString, Symbol, Value};
 use std::ffi::OsString;
 use std::fs::File;
 use std::io::{self, Read, Seek};
@@ -18,7 +18,7 @@ pub struct FileInfo {
     pub path: PathBuf,
     pub len: usize,
     pub multiprocess_mode: Symbol,
-    pub type_: StaticSymbol,
+    pub type_: Symbol,
     pub pid: String,
 }
 
@@ -59,7 +59,7 @@ impl FileInfo {
         let multiprocess_mode = Symbol::from_value(params[1])
             .ok_or_else(|| err!(arg_error(), "expected multiprocess_mode to be a symbol"))?;
 
-        let type_ = StaticSymbol::from_value(params[2])
+        let type_ = Symbol::from_value(params[2])
             .ok_or_else(|| err!(arg_error(), "expected file type to be a symbol"))?;
 
         let pid = RString::from_value(params[3])
@@ -103,7 +103,7 @@ impl FileInfo {
 
 #[cfg(test)]
 mod test {
-    use magnus::{eval, RArray, StaticSymbol, Symbol};
+    use magnus::{eval, RArray, Symbol};
     use rand::{thread_rng, Rng};
     use sha2::{Digest, Sha256};
 
@@ -168,7 +168,7 @@ mod test {
             path,
             len: buf.len(),
             multiprocess_mode: Symbol::new("puma"),
-            type_: StaticSymbol::new("max"),
+            type_: Symbol::new("max"),
             pid: "worker-0_0".to_string(),
         };
 

data/ext/fast_mmaped_file_rs/src/map.rs

@@ -166,7 +166,7 @@ impl EntryMap {
 
 #[cfg(test)]
 mod test {
-    use magnus::{StaticSymbol, Symbol};
+    use magnus::Symbol;
     use std::mem;
 
     use super::*;
@@ -197,7 +197,7 @@ mod test {
                 },
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("max"),
-                    type_: StaticSymbol::new("gauge"),
+                    type_: Symbol::new("gauge"),
                     value: 1.0,
                 },
             },
@@ -208,7 +208,7 @@ mod test {
                 },
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("max"),
-                    type_: StaticSymbol::new("gauge"),
+                    type_: Symbol::new("gauge"),
                     value: 1.0,
                 },
             },
@@ -219,7 +219,7 @@ mod test {
                 },
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("max"),
-                    type_: StaticSymbol::new("gauge"),
+                    type_: Symbol::new("gauge"),
                     value: 1.0,
                 },
             },
@@ -230,7 +230,7 @@ mod test {
                 },
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("max"),
-                    type_: StaticSymbol::new("gauge"),
+                    type_: Symbol::new("gauge"),
                     value: 1.0,
                 },
             },
@@ -241,7 +241,7 @@ mod test {
                 },
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("all"),
-                    type_: StaticSymbol::new("gauge"),
+                    type_: Symbol::new("gauge"),
                     value: 1.0,
                 },
             },
@@ -252,7 +252,7 @@ mod test {
                 },
                 meta: EntryMetadata {
                     multiprocess_mode: Symbol::new("all"),
-                    type_: StaticSymbol::new("gauge"),
+                    type_: Symbol::new("gauge"),
                     value: 1.0,
                 },
             },
@@ -293,7 +293,7 @@ mod test {
             },
             meta: EntryMetadata {
                 multiprocess_mode: Symbol::new("all"),
-                type_: StaticSymbol::new("gauge"),
+                type_: Symbol::new("gauge"),
                 value: 1.0,
             },
         };
@@ -305,7 +305,7 @@ mod test {
             },
             meta: EntryMetadata {
                 multiprocess_mode: Symbol::new("all"),
-                type_: StaticSymbol::new("gauge"),
+                type_: Symbol::new("gauge"),
                 value: 5.0,
             },
         };
@@ -317,7 +317,7 @@ mod test {
             },
             meta: EntryMetadata {
                 multiprocess_mode: Symbol::new("all"),
-                type_: StaticSymbol::new("gauge"),
+                type_: Symbol::new("gauge"),
                 value: 100.0,
             },
         };
@@ -329,7 +329,7 @@ mod test {
             },
             meta: EntryMetadata {
                 multiprocess_mode: Symbol::new("all"),
-                type_: StaticSymbol::new("gauge"),
+                type_: Symbol::new("gauge"),
                 value: 1.0,
             },
         };
@@ -460,7 +460,7 @@ mod test {
                 path,
                 len: case.json.len(),
                 multiprocess_mode: Symbol::new("max"),
-                type_: StaticSymbol::new("gauge"),
+                type_: Symbol::new("gauge"),
                 pid: "worker-1".to_string(),
             };
 

data/ext/fast_mmaped_file_rs/src/mmap/inner.rs

@@ -204,6 +204,12 @@ impl InnerMmap {
         self.map.as_ptr()
     }
 
+    /// Returns a mutable raw pointer to the mmap.
+    /// For use in updating RString internals which requires a mutable pointer.
+    pub fn as_mut_ptr(&self) -> *mut u8 {
+        self.map.as_ptr().cast_mut()
+    }
+
     /// Perform an msync(2) on the mmap, flushing all changes written
     /// to disk. The sync may optionally be performed asynchronously.
     pub fn flush(&mut self, f_async: bool) -> Result<()> {

data/ext/fast_mmaped_file_rs/src/mmap.rs

@@ -3,8 +3,8 @@ use magnus::prelude::*;
 use magnus::rb_sys::{AsRawValue, FromRawValue};
 use magnus::typed_data::Obj;
 use magnus::value::Fixnum;
-use magnus::{block::Proc, eval, scan_args, Error, Integer, RArray, RClass, RHash, RString, Value};
-use nix::libc::c_long;
+use magnus::{eval, scan_args, Error, Integer, RArray, RClass, RHash, RString, Value};
+use nix::libc::{c_char, c_long, c_ulong};
 use rb_sys::rb_str_new_static;
 use std::fs::File;
 use std::io::{prelude::*, SeekFrom};
@@ -25,6 +25,11 @@ use inner::InnerMmap;
 
 mod inner;
 
+/// The Ruby `STR_NOEMBED` flag, aka `FL_USER1`.
+const STR_NOEMBED: c_ulong = 1 << (13);
+/// The Ruby `STR_SHARED` flag, aka `FL_USER2`.
+const STR_SHARED: c_ulong = 1 << (14);
+
 /// A Rust struct wrapped in a Ruby object, providing access to a memory-mapped
 /// file used to store, update, and read out Prometheus metrics.
 ///
@@ -151,13 +156,28 @@ impl MmapedFile {
     ///
     /// return a substring of <em>lenght</em> characters from <em>start</em>
     pub fn slice(rb_self: Obj<Self>, args: &[Value]) -> magnus::error::Result<RString> {
-        // The C implemenation would trigger a GC cycle via `rb_gc_force_recycle`
+        // The C implementation would trigger a GC cycle via `rb_gc_force_recycle`
         // if the `MM_PROTECT` flag is set, but in practice this is never used.
         // We omit this logic, particularly because `rb_gc_force_recycle` is a
         // no-op as of Ruby 3.1.
+        let rs_self = &*rb_self;
+
+        let str = rs_self.str(rb_self)?;
+        rs_self._slice(rb_self, str, args)
+    }
 
-        let str = (*rb_self).str(rb_self)?;
-        let res = str.funcall("[]", args);
+    fn _slice(
+        &self,
+        rb_self: Obj<Self>,
+        str: RString,
+        args: &[Value],
+    ) -> magnus::error::Result<RString> {
+        let substr: RString = str.funcall("[]", args)?;
+
+        // Track shared child strings which use the same backing storage.
+        if Self::rb_string_is_shared(substr) {
+            (*rb_self).track_rstring(rb_self, substr)?;
+        }
 
         // The C implementation does this, perhaps to validate that the len we
         // provided is actually being used.
@@ -166,7 +186,7 @@ impl MmapedFile {
             Ok(())
         })?;
 
-        res
+        Ok(substr)
     }
 
     /// Document-method: msync
@@ -223,7 +243,8 @@ impl MmapedFile {
         })?;
 
         // Update each String object to be zero-length.
-        rs_self.update_weak_map(rb_self)?;
+        let cap = util::cast_chk::<_, c_long>(rs_self.capacity(), "capacity")?;
+        rs_self.update_weak_map(rb_self, rs_self.as_mut_ptr(), cap)?;
 
         // Remove the `InnerMmap` from the `RwLock`. This will drop
         // end of this function, unmapping and closing the file.
@@ -341,16 +362,19 @@ impl MmapedFile {
             Ok(unsafe { rb_str_new_static(ptr as _, len as _) })
         })?;
 
+        // SAFETY: We know that rb_str_new_static returns a VALUE.
         let val = unsafe { Value::from_raw(val_id) };
 
         // UNWRAP: We created this value as a string above.
         let str = RString::from_value(val).unwrap();
 
-        let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
+        // Freeze the root string so it can't be mutated out from under any
+        // substrings created. This object is never exposed to callers.
+        str.freeze();
 
-        // Use the string's Id as the key in the `WeakMap`.
-        let key = str.as_raw();
-        let _: Value = tracker.funcall("[]=", (key, str))?;
+        // Track the RString in our `WeakMap` so we can update its address if
+        // we re-mmap the backing file.
+        (*rb_self).track_rstring(rb_self, str)?;
 
         Ok(str)
     }
@@ -358,23 +382,20 @@ impl MmapedFile {
     /// If we reallocate, any live Ruby strings provided by the `str()` method
     /// will be invalidated. We need to iterate over them using and update their
     /// heap pointers to the newly allocated memory region.
-    fn update_weak_map(&self, rb_self: Obj<Self>) -> magnus::error::Result<()> {
+    fn update_weak_map(
+        &self,
+        rb_self: Obj<Self>,
+        old_ptr: *const c_char,
+        old_cap: c_long,
+    ) -> magnus::error::Result<()> {
         let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
 
-        let (new_ptr, new_len) = (*rb_self).inner(|inner| {
-            // Pointers are not `Send`, convert it to usize to allow capture in closure.
-            let new_ptr = inner.as_ptr() as usize;
-            let new_len = util::cast_chk::<_, c_long>(inner.len(), "mmap len")?;
+        let new_len = self.inner(|inner| util::cast_chk::<_, c_long>(inner.len(), "mmap len"))?;
 
-            Ok((new_ptr, new_len))
-        })?;
-
-        // Allocate a block with a closure that updates the string details.
-        let block = Proc::from_fn(move |args, _block| {
-            let val = args
-                .get(0)
-                .ok_or_else(|| err!(arg_error(), "no argument received from `each_value`"))?;
-            let str = RString::from_value(*val)
+        // Iterate over the values of the `WeakMap`.
+        for val in tracker.enumeratorize("each_value", ()) {
+            let rb_string = val?;
+            let str = RString::from_value(rb_string)
                 .ok_or_else(|| err!(arg_error(), "weakmap value was not a string"))?;
 
             // SAFETY: We're messing with Ruby's internals here, YOLO.
@@ -383,18 +404,42 @@ impl MmapedFile {
                 // which provides access to its internals.
                 let mut raw_str = Self::rb_string_internal(str);
 
+                // Shared string have their own `ptr` and `len` values, but `aux`
+                // is the id of the parent string so the GC can track this
+                // dependency. The `ptr` will always be an offset from the base
+                // address of the mmap, and `len` will be the length of the mmap
+                // less the offset from the base.
+                if Self::rb_string_is_shared(str) && new_len > 0 {
+                    // Calculate how far into the original mmap the shared string
+                    // started and update to the equivalent address in the new
+                    // one.
+                    let substr_ptr = raw_str.as_ref().as_.heap.ptr;
+                    let offset = substr_ptr.offset_from(old_ptr);
+
+                    raw_str.as_mut().as_.heap.ptr = self.as_mut_ptr().offset(offset);
+
+                    let current_len = raw_str.as_ref().as_.heap.len;
+                    let new_shared_len = old_cap + current_len;
+
+                    raw_str.as_mut().as_.heap.len = new_shared_len;
+                    continue;
+                }
+
                 // Update the string to point to the new mmapped file.
                 // We're matching the behavior of Ruby's `str_new_static` function.
                 // See https://github.com/ruby/ruby/blob/e51014f9c05aa65cbf203442d37fef7c12390015/string.c#L1030-L1053
-                raw_str.as_mut().as_.heap.ptr = new_ptr as _;
+                //
+                // We deliberately do _NOT_ increment the `capa` field of the
+                // string to match the new `len`. We were initially doing this,
+                // but consistently triggered GCs in the middle of updating the
+                // string pointers, causing a segfault.
+                //
+                // See https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/issues/45
+                raw_str.as_mut().as_.heap.ptr = self.as_mut_ptr();
                 raw_str.as_mut().as_.heap.len = new_len;
-                raw_str.as_mut().as_.heap.aux.capa = new_len;
             }
-            Ok(())
-        });
+        }
 
-        // Execute the block.
-        let _: Value = tracker.funcall_with_block("each_value", (), block)?;
         Ok(())
     }
 
@@ -429,6 +474,9 @@ impl MmapedFile {
         }
 
         if new_cap != self.capacity() {
+            let old_ptr = self.as_mut_ptr();
+            let old_cap = util::cast_chk::<_, c_long>(self.capacity(), "capacity")?;
+
             // Drop the old mmap.
             let (mut file, path) = self.take_inner()?.munmap();
 
@@ -439,7 +487,7 @@ impl MmapedFile {
 
             self.insert_inner(new_inner)?;
 
-            return self.update_weak_map(rb_self);
+            return self.update_weak_map(rb_self, old_ptr, old_cap);
         }
 
         Ok(())
@@ -476,6 +524,15 @@ impl MmapedFile {
         Ok(())
     }
 
+    fn track_rstring(&self, rb_self: Obj<Self>, str: RString) -> magnus::error::Result<()> {
+        let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
+
+        // Use the string's Id as the key in the `WeakMap`.
+        let key = str.as_raw();
+        let _: Value = tracker.funcall("[]=", (key, str))?;
+        Ok(())
+    }
+
     /// The total capacity of the underlying mmap.
     #[inline]
     fn capacity(&self) -> usize {
@@ -489,6 +546,13 @@ impl MmapedFile {
             .map_err(|e| e.into())
     }
 
+    fn as_mut_ptr(&self) -> *mut c_char {
+        // UNWRAP: This is actually infallible, but we need to
+        // wrap it in a `Result` for use with `inner()`.
+        self.inner(|inner| Ok(inner.as_mut_ptr() as *mut c_char))
+            .unwrap()
+    }
+
     /// Takes a closure with immutable access to InnerMmap. Will fail if the inner
     /// object has a mutable borrow or has been dropped.
     fn inner<F, T>(&self, func: F) -> Result<T>
@@ -544,9 +608,27 @@ impl MmapedFile {
         Ok(())
     }
 
+    /// Check if an RString is shared. Shared string use the same underlying
+    /// storage as their parent, taking an offset from the start. By default
+    /// they must run to the end of the parent string.
+    fn rb_string_is_shared(rb_str: RString) -> bool {
+        // SAFETY: We only hold a reference to the raw object for the duration
+        // of this function, and no Ruby code is called.
+        let flags = unsafe {
+            let raw_str = Self::rb_string_internal(rb_str);
+            raw_str.as_ref().basic.flags
+        };
+        let shared_flags = STR_SHARED | STR_NOEMBED;
+
+        flags & shared_flags == shared_flags
+    }
+
     /// Convert `magnus::RString` into the raw binding used by `rb_sys::RString`.
     /// We need this to manually change the pointer and length values for strings
     /// when moving the mmap to a new file.
+    ///
+    /// SAFETY: Calling Ruby code while the returned object is held may result
+    /// in it being mutated or dropped.
     unsafe fn rb_string_internal(rb_str: RString) -> NonNull<rb_sys::RString> {
         mem::transmute::<RString, NonNull<rb_sys::RString>>(rb_str)
     }
@@ -666,18 +748,101 @@ mod test {
         let ruby = magnus::Ruby::get().unwrap();
         crate::init(&ruby).unwrap();
 
+        fn assert_internals(
+            obj: Obj<MmapedFile>,
+            parent_id: c_ulong,
+            child_id: c_ulong,
+            unshared_id: c_ulong,
+        ) {
+            let rs_self = &*obj;
+            let tracker: Value = obj.ivar_get("@weak_obj_tracker").unwrap();
+
+            let mmap_ptr = rs_self.as_mut_ptr();
+            let mmap_len = rs_self.capacity();
+
+            let mut parent_checked = false;
+            let mut child_checked = false;
+
+            for val in tracker.enumeratorize("each_value", ()) {
+                let rb_string = val.unwrap();
+                let str = RString::from_value(rb_string).unwrap();
+
+                unsafe {
+                    let raw_str = MmapedFile::rb_string_internal(str);
+                    if str.as_raw() == child_id {
+                        assert_eq!(parent_id, raw_str.as_ref().as_.heap.aux.shared);
+
+                        let child_offset =
+                            mmap_len as isize - raw_str.as_ref().as_.heap.len as isize;
+                        assert_eq!(mmap_ptr.offset(child_offset), raw_str.as_ref().as_.heap.ptr);
+
+                        child_checked = true;
+                    } else if str.as_raw() == parent_id {
+                        assert_eq!(parent_id, str.as_raw());
+
+                        assert_eq!(mmap_ptr, raw_str.as_ref().as_.heap.ptr);
+                        assert_eq!(mmap_len as c_long, raw_str.as_ref().as_.heap.len);
+                        assert!(raw_str.as_ref().basic.flags & (STR_SHARED | STR_NOEMBED) > 0);
+                        assert!(str.is_frozen());
+
+                        parent_checked = true;
+                    } else if str.as_raw() == unshared_id {
+                        panic!("tracking unshared string");
+                    } else {
+                        panic!("unknown string");
+                    }
+                }
+            }
+            assert!(parent_checked && child_checked);
+        }
+
         let obj = create_obj();
         let _ = populate_entries(&obj);
 
         let rs_self = &*obj;
 
-        let value_range = Range::new(HEADER_SIZE, 24, false).unwrap().as_value();
-        let value_slice = MmapedFile::slice(obj, &[value_range]).unwrap();
-
-        rs_self.expand_to_fit(obj, rs_self.capacity() * 2).unwrap();
+        // Create a string containing the full mmap.
+        let parent_str = rs_self.str(obj).unwrap();
+        let parent_id = parent_str.as_raw();
+
+        // Ruby's shared strings are only created when they go to the end of
+        // original string.
+        let len = rs_self.inner(|inner| Ok(inner.len())).unwrap();
+        let shareable_range = Range::new(1, len - 1, false).unwrap().as_value();
+
+        // This string should re-use the parent's buffer with an offset and have
+        // the parent's id in `as.heap.aux.shared`
+        let child_str = rs_self._slice(obj, parent_str, &[shareable_range]).unwrap();
+        let child_id = child_str.as_raw();
+
+        // A range that does not reach the end of the parent will not be shared.
+        assert!(len > 4);
+        let unshareable_range = Range::new(0, 4, false).unwrap().as_value();
+
+        // This string should NOT be tracked, it should own its own buffer.
+        let unshared_str = rs_self
+            ._slice(obj, parent_str, &[unshareable_range])
+            .unwrap();
+        let unshared_id = unshared_str.as_raw();
+        assert!(!MmapedFile::rb_string_is_shared(unshared_str));
+
+        assert_internals(obj, parent_id, child_id, unshared_id);
+
+        let orig_ptr = rs_self.as_mut_ptr();
+        // Expand a bunch to ensure we remap
+        for _ in 0..16 {
+            rs_self.expand_to_fit(obj, rs_self.capacity() * 2).unwrap();
+        }
+        let new_ptr = rs_self.as_mut_ptr();
+        assert!(orig_ptr != new_ptr);
 
         // If we haven't updated the pointer to the newly remapped file this will segfault.
-        let _: Value = eval!("puts str", str = value_slice).unwrap();
+        let _: Value = eval!("puts parent", parent = parent_str).unwrap();
+        let _: Value = eval!("puts child", child = child_str).unwrap();
+        let _: Value = eval!("puts unshared", unshared = unshared_str).unwrap();
+
+        // Confirm that tracked strings are still valid.
+        assert_internals(obj, parent_id, child_id, unshared_id);
     }
 
     #[test]

data/ext/fast_mmaped_file_rs/src/parser.rs

@@ -2,7 +2,7 @@ use smallvec::SmallVec;
 use std::str;
 
 /// String slices pointing to the fields of a borrowed `Entry`'s JSON data.
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Eq, Debug)]
 pub struct MetricText<'a> {
     pub family_name: &'a str,
     pub metric_name: &'a str,
@@ -10,14 +10,14 @@ pub struct MetricText<'a> {
     pub values: SmallVec<[&'a str; 4]>,
 }
 
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Eq, Debug)]
 struct MetricNames<'a> {
     label_json: &'a str,
     family_name: &'a str,
     metric_name: &'a str,
 }
 
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Eq, Debug)]
 struct MetricLabelVals<'a> {
     labels: SmallVec<[&'a str; 4]>,
     values: SmallVec<[&'a str; 4]>,
@@ -63,35 +63,35 @@ fn parse_names(json: &str) -> Option<MetricNames> {
         return None;
     }
 
-    // Now: family_name","metric_name",[...
-    let remainder = json.get(2..)?;
-
-    let names_end = remainder.find('[')?;
-
-    // Save the rest of the slice to parse for labels later.
-    let label_json = remainder.get(names_end..)?;
-
-    //  Now: family_name","metric_name",
-    let remainder = remainder.get(..names_end)?;
+    // Captured: "family_name","metric_name",
+    //            ^^^^^^^^^^^
+    let (family_name, remainder) = scan_str(json.get(1..)?)?;
 
-    // Split on commas into:
-    // family_name","metric_name",
-    // ^^^^one^^^^^ ^^^^^two^^^^^
-    let mut token_iter = remainder.split(',');
+    //  Validate comma separated names
+    // ["family_name","metric_name",[...
+    //               ^
+    if !remainder.starts_with("\",") {
+        return None;
+    }
 
-    // Captured: family_name","metric_name",
-    //           ^^^^^^^^^^^
-    let family_name = token_iter.next()?.trim_end_matches('"');
+    // Now: "metric_name",[...
+    let remainder = remainder.get(2..)?;
 
     // Captured: "family_name","metric_name",
     //                          ^^^^^^^^^^^
-    let metric_name = token_iter.next()?.trim_matches('"');
+    let (metric_name, remainder) = scan_str(remainder)?;
 
-    // Confirm the final entry of the iter is empty, the the trailing ','.
-    if !token_iter.next()?.is_empty() {
+    //  Validate comma separated names
+    // ["family_name","metric_name",[...
+    //                            ^^
+    if !remainder.starts_with("\",") {
         return None;
     }
 
+    // Save the rest of the slice to parse for labels later.
+    // Now: [...
+    let label_json = remainder.get(2..)?;
+
     Some(MetricNames {
         label_json,
         family_name,
@@ -101,76 +101,143 @@ fn parse_names(json: &str) -> Option<MetricNames> {
 
 fn parse_label_values(json: &str) -> Option<MetricLabelVals> {
     // Starting with: ["label_a","label_b"],["value_a", "value_b"]]
-    if !(json.starts_with('[') && json.ends_with("]]")) {
+    if !json.starts_with('[') {
         return None;
     }
 
+    // Now: "label_a","label_b"],["value_a", "value_b"]]
+    let mut remainder = json.get(1..)?;
+
     // Validate we either have the start of a label string or an
     // empty array, e.g. `["` or `[]`.
     if !matches!(json.as_bytes().get(1)?, b'"' | b']') {
         return None;
     }
 
-    // Now: "label_a","label_b"
-    let labels_end = json.find(']')?;
-    let label_range = json.get(1..labels_end)?;
-
     let mut labels = SmallVec::new();
 
     // Split on commas into:
     // "label_a","label_b"
     // ^^^one^^^ ^^^two^^^
-    for label in label_range.split(',') {
-        // Captured: "label_a","label_b"
-        //            ^^^^^^^
-        // If there are no labels, e.g. `[][]`, then don't capture anything.
-        if !label.is_empty() {
-            labels.push(label.trim_matches('"'));
-        }
+    loop {
+        let Some((label, label_rem)) = scan_str(remainder) else {
+            // No further keys.
+            break;
+        };
+        labels.push(label);
+
+        // Advance past trailing quote.
+        remainder = label_rem.get(1..)?;
+        match remainder.as_bytes().first() {
+            Some(b']') => break, // No further labels.
+            Some(b',') => {}     // More labels expected.
+            _ => return None,    // Invalid.
+        };
+
+        // Advance past comma.
+        remainder = remainder.get(1..)?;
     }
 
-    // Now: ],["value_a", "value_b"]]
-    let mut values_range = json.get(labels_end..)?;
-
-    // Validate we have a separating comma with one and only one leading bracket.
-    if !(values_range.starts_with("],[") && values_range.as_bytes().get(3)? != &b'[') {
+    if !remainder.starts_with("],[") {
         return None;
     }
-
     // Now: "value_a", "value_b"]]
-    values_range = values_range.get(3..)?;
-
-    let values_end = values_range.find(']')?;
+    remainder = remainder.get(3..)?;
 
-    // Validate we have only two trailing brackets.
-    if values_range.get(values_end..)?.len() > 2 {
+    // Validate we don't have extra brackets.
+    if remainder.starts_with('[') {
         return None;
     }
 
-    // Now: "value_a", "value_b"
-    values_range = values_range.get(..values_end)?;
-
     let mut values = SmallVec::new();
-
     // Split on commas into:
     // "value_a","value_b"
     // ^^^one^^^ ^^^two^^^
-    for value in values_range.split(',') {
-        // Captured: "value_a","value_b"
-        //           ^^^^^^^^^
-        // If there are no values, e.g. `[][]`, then don't capture anything.
-        if !value.is_empty() {
-            values.push(value.trim_matches('"'));
+    loop {
+        if remainder.starts_with('"') {
+            let (value, value_rem) = scan_str(remainder)?;
+            values.push(value);
+
+            // Advance past trailing quote.
+            remainder = value_rem.get(1..)?;
+        } else {
+            // An unquoted value, such as `true` or `404`.
+            let i = remainder.find(|c| c == ',' || c == ']')?;
+            let value = &remainder[..i];
+
+            match (value.is_empty(), is_valid_json_literal(value)) {
+                (true, _) => {}                      // Empty string, do nothing.
+                (false, true) => values.push(value), // Valid literal.
+                (false, false) => return None,       // Invalid literal, fail.
+            }
+
+            remainder = &remainder[i..];
         }
+
+        match remainder.as_bytes().first() {
+            Some(b']') => break, // End of values.
+            Some(b',') => {}     // More values expected.
+            _ => return None,    // Invalid.
+        };
+
+        // Advance past comma.
+        remainder = remainder.get(1..)?;
     }
 
     if values.len() != labels.len() {
         return None;
     }
 
+    // Now: ]]
+    if remainder != "]]" {
+        return None;
+    }
+
     Some(MetricLabelVals { labels, values })
 }
 
+// Read a JSON-encoded str that includes starting and ending double quotes.
+// Returns the validated str with the double quotes trimmed and the remainder
+// of the input str.
+fn scan_str(json: &str) -> Option<(&str, &str)> {
+    let mut escaping = false;
+
+    if !json.starts_with('"') {
+        return None;
+    }
+
+    // Trim leading double quote.
+    let json = json.get(1..)?;
+
+    for (i, &c) in json.as_bytes().iter().enumerate() {
+        if c == b'\\' {
+            escaping = true;
+            continue;
+        }
+
+        if c == b'"' && !escaping {
+            return Some((json.get(..i)?, json.get(i..)?));
+        }
+
+        escaping = false;
+    }
+
+    None
+}
+
+// Confirm an unquoted JSON literal is a boolean, null, or has a passing
+// resemblance to a number. We do not confirm numeric formatting, only
+// that all characters are valid. See https://www.json.org/json-en.html
+// for details.
+fn is_valid_json_literal(s: &str) -> bool {
+    match s {
+        "true" | "false" | "null" => true,
+        _ => s.chars().all(|c| {
+            c.is_ascii_digit() || c == '.' || c == '+' || c == '-' || c == 'e' || c == 'E'
+        }),
+    }
+}
+
 #[cfg(test)]
 mod test {
     use smallvec::smallvec;
@@ -217,6 +284,16 @@ mod test {
                     values: smallvec!["value_a", "403"],
                 }),
             },
+            TestCase {
+                name: "scientific notation literal",
+                input: r#"["metric","name",["label_a"],[-2.0e-5]]"#,
+                expected: Some(MetricText {
+                    family_name: "metric",
+                    metric_name: "name",
+                    labels: smallvec!["label_a"],
+                    values: smallvec!["-2.0e-5"],
+                }),
+            },
             TestCase {
                 name: "null value",
                 input: r#"["metric","name",["label_a","label_b"],[null,"value_b"]]"#,
@@ -237,6 +314,36 @@ mod test {
                     values: smallvec![],
                 }),
             },
+            TestCase {
+                name: "comma in items",
+                input: r#"["met,ric","na,me",["label,_a","label_b"],["value,_a","value_b"]]"#,
+                expected: Some(MetricText {
+                    family_name: "met,ric",
+                    metric_name: "na,me",
+                    labels: smallvec!["label,_a", "label_b"],
+                    values: smallvec!["value,_a", "value_b"],
+                }),
+            },
+            TestCase {
+                name: "bracket in value",
+                input: r#"["met[r]ic","na[m]e",["label[_]a","label_b"],["value_a","val[ue]_b"]]"#,
+                expected: Some(MetricText {
+                    family_name: "met[r]ic",
+                    metric_name: "na[m]e",
+                    labels: smallvec!["label[_]a", "label_b"],
+                    values: smallvec!["value_a", "val[ue]_b"],
+                }),
+            },
+            TestCase {
+                name: "escaped quotes",
+                input: r#"["met\"ric","na\"me",["label\"_a","label_b"],["value\"_a","value_b"]]"#,
+                expected: Some(MetricText {
+                    family_name: r#"met\"ric"#,
+                    metric_name: r#"na\"me"#,
+                    labels: smallvec![r#"label\"_a"#, "label_b"],
+                    values: smallvec![r#"value\"_a"#, "value_b"],
+                }),
+            },
         ];
 
         for case in tc {
@@ -313,23 +420,18 @@ mod test {
                 expected: None,
             },
             TestCase {
-                name: "comma in family name",
-                input: r#"["met,ric","name",["label_a","label_b"],["value_a","value_b"]]"#,
-                expected: None,
-            },
-            TestCase {
-                name: "comma in metric name",
-                input: r#"["metric","na,me",["label_a","label_b"],["value_a","value_b"]]"#,
+                name: "misplaced bracket",
+                input: r#"["metric","name",["label_a","label_b"],]["value_a","value_b"]]"#,
                 expected: None,
             },
             TestCase {
-                name: "comma in value",
-                input: r#"["metric","na,me",["label_a","label_b"],["val,ue_a","value_b"]]"#,
+                name: "comma in numeric value",
+                input: r#"["metric","name",["label_a","label_b"],[400,0,"value_b"]]"#,
                 expected: None,
             },
             TestCase {
-                name: "comma in numeric value",
-                input: r#"["metric","name",["label_a","label_b"],[400,0,"value_b"]]"#,
+                name: "non-e letter in numeric value",
+                input: r#"["metric","name",["label_a","label_b"],[400x0,"value_b"]]"#,
                 expected: None,
             },
         ];

data/ext/fast_mmaped_file_rs/src/raw_entry.rs

@@ -6,7 +6,7 @@ use crate::util::CheckedOps;
 use crate::Result;
 
 /// The logic to save a `MetricsEntry`, or parse one from a byte slice.
-#[derive(PartialEq, Clone, Debug)]
+#[derive(PartialEq, Eq, Clone, Debug)]
 pub struct RawEntry<'a> {
     bytes: &'a [u8],
     encoded_len: usize,

data/ext/fast_mmaped_file_rs/src/testhelper.rs

@@ -73,7 +73,7 @@ pub fn entries_to_db(entries: &[&'static str], values: &[f64], header: Option<u3
     };
 
     out.extend(used.to_ne_bytes());
-    out.extend(&[0x0u8; 4]); // Padding.
+    out.extend([0x0u8; 4]); // Padding.
     out.extend(entry_bytes);
 
     out

data/lib/prometheus/client/configuration.rb

@@ -7,13 +7,14 @@ require 'tmpdir'
 module Prometheus
   module Client
     class Configuration
-      attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider
+      attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider, :rust_multiprocess_metrics
 
       def initialize
         @value_class = ::Prometheus::Client::MmapedValue
         @initial_mmap_file_size = ::Prometheus::Client::PageSize.page_size(fallback_page_size: 4096)
         @logger = Logger.new($stdout)
         @pid_provider = Process.method(:pid)
+        @rust_multiprocess_metrics = ENV.fetch('prometheus_rust_multiprocess_metrics', nil) == 'true'
         @multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do
           Dir.mktmpdir("prometheus-mmap")
         end

data/lib/prometheus/client/formats/text.rb

@@ -1,5 +1,6 @@
 require 'prometheus/client/uses_value_type'
 require 'prometheus/client/helper/json_parser'
+require 'prometheus/client/helper/loader'
 require 'prometheus/client/helper/plain_file'
 require 'prometheus/client/helper/metrics_processing'
 require 'prometheus/client/helper/metrics_representation'
@@ -31,7 +32,7 @@ module Prometheus
               .map {|f| Helper::PlainFile.new(f) }
               .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
 
-            if use_rust && rust_impl_available?
+            if use_rust && Prometheus::Client::Helper::Loader.rust_impl_available?
               FastMmapedFileRs.to_metrics(file_list.to_a)
             else
               FastMmapedFile.to_metrics(file_list.to_a)
@@ -46,29 +47,6 @@ module Prometheus
 
           private
 
-          def load_rust_extension
-            begin
-              ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
-              require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
-            rescue LoadError
-              require 'fast_mmaped_file_rs'
-            end
-          end
-
-          def check_for_rust
-            # This will be evaluated on each invocation even with `||=` if
-            # `@rust_available` if false. Running a `require` statement is slow,
-            # so the `rust_impl_available?` method memoizes the result, external
-            # callers can only trigger this method a single time.
-            @rust_available = begin
-              load_rust_extension
-              true
-            rescue LoadError
-              Prometheus::Client.logger.info('FastMmapedFileRs unavailable')
-              false
-            end
-          end
-
           def load_metrics(path)
             metrics = {}
             Dir.glob(File.join(path, '*.db')).sort.each do |f|

data/lib/prometheus/client/helper/loader.rb

@@ -0,0 +1,40 @@
+module Prometheus
+  module Client
+    module Helper
+      module Loader
+        class << self
+          def rust_impl_available?
+            return @rust_available unless @rust_available.nil?
+
+            check_for_rust
+          end
+
+          private
+
+          def load_rust_extension
+            begin
+              ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
+              require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
+            rescue LoadError
+              require 'fast_mmaped_file_rs'
+            end
+          end
+
+          def check_for_rust
+            # This will be evaluated on each invocation even with `||=` if
+            # `@rust_available` if false. Running a `require` statement is slow,
+            # so the `rust_impl_available?` method memoizes the result, external
+            # callers can only trigger this method a single time.
+            @rust_available = begin
+              load_rust_extension
+              true
+            rescue LoadError
+              Prometheus::Client.logger.info('FastMmapedFileRs unavailable')
+              false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/mmaped_file.rb

@@ -1,5 +1,6 @@
 require 'prometheus/client/helper/entry_parser'
 require 'prometheus/client/helper/file_locker'
+require 'prometheus/client/helper/loader'
 
 # load precompiled extension if available
 begin
@@ -12,7 +13,17 @@ end
 module Prometheus
   module Client
     module Helper
-      class MmapedFile < FastMmapedFile
+      # We can't check `Prometheus::Client.configuration` as this creates a circular dependency
+      if (ENV.fetch('prometheus_rust_mmaped_file', nil) == "true" &&
+          Prometheus::Client::Helper::Loader.rust_impl_available?)
+        class MmapedFile < FastMmapedFileRs
+        end
+      else
+        class MmapedFile < FastMmapedFile
+        end
+      end
+
+      class MmapedFile
         include EntryParser
 
         attr_reader :filepath, :size

data/lib/prometheus/client/rack/exporter.rb

@@ -62,8 +62,10 @@ module Prometheus
         end
 
         def respond_with(format)
+          rust_enabled = Prometheus::Client.configuration.rust_multiprocess_metrics
+
           response = if Prometheus::Client.configuration.value_class.multiprocess
-                       format.marshal_multiprocess
+                       format.marshal_multiprocess(use_rust: rust_enabled)
                      else
                        format.marshal
                      end

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '0.23.1'.freeze
+    VERSION = '0.24.4'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 0.23.1
+  version: 0.24.4
 platform: x86_64-darwin
 authors:
 - Tobias Schmidt
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-05-02 00:00:00.000000000 Z
+date: 2023-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -180,6 +180,7 @@ files:
 - lib/prometheus/client/helper/entry_parser.rb
 - lib/prometheus/client/helper/file_locker.rb
 - lib/prometheus/client/helper/json_parser.rb
+- lib/prometheus/client/helper/loader.rb
 - lib/prometheus/client/helper/metrics_processing.rb
 - lib/prometheus/client/helper/metrics_representation.rb
 - lib/prometheus/client/helper/mmaped_file.rb