prometheus-client-mmap 0.28.1-x86_64-linux → 1.0.0-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '014973ac0ab1c0e391251b9a9731687a5f178e91a738901d47331434fc4340fb'
-  data.tar.gz: 2b5c20d70ce59bef31e29bb1779c9c43c0a6428ecbe0099d22e187b27bf0cbf3
+  metadata.gz: 73360e69c9b83ad6a3748094621dc3f93d5fc9d9ff79e881f320bcbb35d9f798
+  data.tar.gz: 6a9fa48dbb5da204fe5bf5007ec8a24f4be3b3ba5290344c635027b14e997a0d
 SHA512:
-  metadata.gz: d2e5478ac79c584831f078d33fbc3a6bac17602ec54a1fdd64d52b2f6edc05b29fa98916d96a8716c6bcfc8b69975b400c6713ec7fed32a7fff5e0ec2106e22c
-  data.tar.gz: 9295e5754f4eb3c8ce0ff8828142cde72483b89b8751adabcdd6aaaaafa5fc23c19eddc4344a4f10b25320be06004d34a8989b6b51b3e37e19fe052597d73732
+  metadata.gz: decae167312a1f71047a6a830dc25e6f497f0c537657749c597720f548462ec250c32b44f09f345338b2bde9c3d16d3131f9f8217cb606583f1e2ddd2d93d9f8
+  data.tar.gz: 426245645098e4296757e9eeb23ebc807fef93b954c92b68b8d678f358fc7540f69bc822dc7d171a6244dba5d686c6b9ccc12ea647da33f142075d05de07c09d

data/.tool-versions

@@ -1 +1 @@
-rust 1.65.0
+rust 1.73.0

data/README.md

@@ -53,19 +53,9 @@ http_requests.increment
 
 ## Rust extension
 
-In an effort to improve maintainability, there is now a Rust
+This gem now uses a rewritten Rust extension instead of C.
 implementation that reads the metric files and outputs the multiprocess
-metrics to text. If `rustc` is available, then the Rust extension will
-be built automatically. The `use_rust` keyword argument can be used:
-
-```ruby
-puts Prometheus::Client::Formats::Text.marshal_multiprocess(use_rust: true)
-```
-
-Starting with v0.26.0, `use_rust` defaults to `true`.
-
-Note that this parameter will likely be deprecated and removed once the Rust
-extension becomes the only supported mode.
+metrics to text. This implementation is significantly faster than the C extension.
 
 ### Rack middleware
 
@@ -219,24 +209,6 @@ Set `prometheus_multiproc_dir` environment variable to the path where you want m
 prometheus_multiproc_dir=/tmp
 ```
 
-### Multiprocess metrics via Rust extension
-
-By default, the Rust extension will be used to read and write multiprocess
-metrics if the `fast_mmaped_file_rs` extension is available. This should
-be significantly faster than the C extension.
-
-If the environment variable `prometheus_rust_multiprocess_metrics=false`
-is set or if the `rust_multiprocess_metrics` configuration setting is
-`false` , the C extension will be used.
-
-### Read and write metrics via Rust extension
-
-By default, the Rust extension will be used to read and write metrics
-from the mmaped file if the `fast_mmaped_file_rs` extension is available.
-
-To use the C extension, set the environment variable
-`prometheus_rust_mmaped_file=false`.
-
 ## Pitfalls
 
 ### PID cardinality

data/ext/fast_mmaped_file_rs/extconf.rb

@@ -24,7 +24,5 @@ if find_executable('rustc')
     ARGV = trimmed_argv
   end
 else
-  puts 'rustc not found, skipping Rust extension.'
-
-  File.write('Makefile', dummy_makefile($srcdir).join(''))
+  raise 'rustc not found. prometheus-client-mmap now requires Rust.'
 end

data/ext/fast_mmaped_file_rs/src/error.rs

@@ -100,6 +100,13 @@ pub enum MmapError {
     /// No mmap open.
     #[error("unmapped file")]
     UnmappedFile,
+    /// File length changed by another process.
+    #[error("size of {path} {len} does not match previous size {expected_len}. Another process may have modified it.")]
+    UnexpectedFileLength {
+        path: String,
+        len: usize,
+        expected_len: usize,
+    },
     /// A custom error message with `strerror(3)` appended.
     #[error("{0}")]
     WithErrno(String),
@@ -162,6 +169,7 @@ impl MmapError {
             MmapError::Other(_) => RubyError::Arg,
             MmapError::PromParsing(_) => RubyError::PromParsing,
             MmapError::UnmappedFile => RubyError::Io,
+            MmapError::UnexpectedFileLength { .. } => RubyError::Runtime,
             MmapError::WithErrno(_) => RubyError::Io,
         }
     }

data/ext/fast_mmaped_file_rs/src/mmap/inner.rs

@@ -5,7 +5,7 @@ use std::fs::File;
 use std::mem::size_of;
 use std::ops::Range;
 use std::os::unix::prelude::{AsRawFd, RawFd};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use crate::error::{MmapError, RubyError};
 use crate::raw_entry::RawEntry;
@@ -34,6 +34,8 @@ pub(super) struct InnerMmap {
     ///
     /// Equivalent to `i_mm->t->real` in the C implementation.
     len: usize,
+    /// The length of the file being mmapped at the time of mapping.
+    file_len: usize,
 }
 
 impl InnerMmap {
@@ -41,19 +43,12 @@ impl InnerMmap {
     /// Use when mmapping a file for the first time. When re-mapping a file
     /// after expanding it the `reestablish` function should be used.
     pub fn new(path: PathBuf, file: File) -> Result<Self> {
-        let stat = file.metadata().map_err(|e| {
-            MmapError::legacy(
-                format!("Can't stat {}: {e}", path.display()),
-                RubyError::Arg,
-            )
-        })?;
-
-        let file_size = util::cast_chk::<_, usize>(stat.len(), "file length")?;
+        let initial_file_len = get_file_len(&path, &file)?;
 
         // We need to ensure the underlying file descriptor is at least a page size.
         // Otherwise, we could get a SIGBUS error if mmap() attempts to read or write
         // past the file.
-        let reserve_size = Self::next_page_boundary(file_size)?;
+        let reserve_size = Self::next_page_boundary(initial_file_len)?;
 
         // Cast: no-op.
         Self::reserve_mmap_file_bytes(file.as_raw_fd(), reserve_size as off_t).map_err(|e| {
@@ -67,21 +62,23 @@ impl InnerMmap {
         })?;
 
         // Ensure we always have space for the header.
-        let map_len = file_size.max(HEADER_SIZE);
+        let map_len = initial_file_len.max(HEADER_SIZE);
 
         // SAFETY: There is the possibility of UB if the file is modified outside of
         // this program.
         let map = unsafe { MmapOptions::new().len(map_len).map_mut(&file) }.map_err(|e| {
             MmapError::legacy(format!("mmap failed ({}): {e}", errno()), RubyError::Arg)
         })?;
+        let file_len = util::cast_chk::<_, usize>(reserve_size, "file length")?;
 
-        let len = file_size;
+        let len = initial_file_len;
 
         Ok(Self {
             file,
             path,
             map,
             len,
+            file_len,
         })
     }
 
@@ -97,12 +94,14 @@ impl InnerMmap {
         // to the old length at this point if closing the file. Matching C implementation
         // for now.
         let len = map_len;
+        let file_len = get_file_len(&path, &file)?;
 
         Ok(Self {
             file,
             path,
             map,
             len,
+            file_len,
         })
     }
 
@@ -186,7 +185,7 @@ impl InnerMmap {
         self.len
     }
 
-    /// The total length in bytes of the mmapped file.
+    /// The total length in bytes of the mmapped region of the file.
     ///
     /// Equivalent to `i_mm->t->len` in the C implementation.
     #[inline]
@@ -194,6 +193,21 @@ impl InnerMmap {
         self.map.len()
     }
 
+    /// `stat` the mmapped file and return its length.
+    pub fn check_file_len(&self) -> Result<usize> {
+        get_file_len(&self.path, &self.file)
+    }
+
+    /// The length of the file at the time it was mmapped.
+    pub fn file_len(&self) -> usize {
+        self.file_len
+    }
+
+    /// The path of the mmapped file.
+    pub fn path(&self) -> String {
+        self.path.to_string_lossy().to_string()
+    }
+
     /// Update the length of the mmap considered to be written.
     pub fn set_len(&mut self, len: usize) {
         self.len = len;
@@ -312,6 +326,17 @@ impl InnerMmap {
     }
 }
 
+fn get_file_len(path: &Path, file: &File) -> Result<usize> {
+    let stat = file.metadata().map_err(|e| {
+        MmapError::legacy(
+            format!("Can't stat {}: {e}", path.display()),
+            RubyError::Arg,
+        )
+    })?;
+
+    util::cast_chk::<_, usize>(stat.len(), "file length")
+}
+
 #[cfg(test)]
 mod test {
     use nix::unistd::{self, SysconfVar};

data/ext/fast_mmaped_file_rs/src/mmap.rs

@@ -441,17 +441,49 @@ impl MmapedFile {
     }
 
     /// Check that the mmap is large enough to contain the value to be added,
-    /// and expand it to fit if necessary.
+    /// and expand it to fit if necessary. Will return an error if the file
+    /// has been resized by another process.
     fn check_expand(&self, rb_self: Obj<Self>, key_len: usize) -> magnus::error::Result<()> {
         // CAST: no-op on 32-bit, widening on 64-bit.
         let used = self.inner(|inner| inner.load_used())? as usize;
         let entry_len = RawEntry::calc_total_len(key_len)?;
+        let required_len = used.add_chk(entry_len)?;
+        let mut target_cap = self.capacity();
 
+        let mut needs_resize = false;
         // We need the mmapped region to contain at least one byte beyond the
-        // written data to create a NUL- terminated C string. Validate that
-        // new length does not exactly match or exceed the length of the mmap.
-        while self.capacity() <= used.add_chk(entry_len)? {
-            self.expand_to_fit(rb_self, self.capacity().mul_chk(2)?)?;
+        // written data to create a NUL-terminated C string. Validate that
+        // new length does not exactly match the length of the mmap.
+        if target_cap <= required_len {
+            needs_resize = true;
+            target_cap = required_len + 1;
+        }
+
+        let (current_file_len, original_file_len) =
+            self.inner(|inner| Ok((inner.check_file_len()?, inner.file_len())))?;
+
+        let file_modified = current_file_len != original_file_len;
+        if file_modified {
+            // Ensure our new capacity is at least as large as the actual file.
+            // Erroring out without first resizing the mmap would leave us stuck
+            // in a bad state.
+            target_cap = current_file_len
+        }
+
+        if needs_resize {
+            self.expand_to_fit(rb_self, target_cap)?;
+        }
+
+        if file_modified {
+            // Return an error so the caller is alerted that there were concurrent writes to the
+            // file.
+            let path = self.inner(|inner| Ok(inner.path()))?;
+            return Err(MmapError::UnexpectedFileLength {
+                path,
+                len: current_file_len,
+                expected_len: original_file_len,
+            }
+            .into());
         }
 
         Ok(())
@@ -652,6 +684,7 @@ mod test {
     use super::*;
     use crate::raw_entry::RawEntry;
     use crate::testhelper::TestFile;
+    use std::os::unix::prelude::AsRawFd;
 
     /// Create a wrapped MmapedFile object.
     fn create_obj() -> Obj<MmapedFile> {
@@ -881,4 +914,37 @@ mod test {
         // Validate that we have expanded the mmap, ensuring a trailing NUL.
         assert!(rs_self.capacity() > current_cap);
     }
+
+    #[test]
+    fn test_detect_external_resize() {
+        let _cleanup = unsafe { magnus::embed::init() };
+        let ruby = magnus::Ruby::get().unwrap();
+        crate::init(&ruby).unwrap();
+
+        let TestFile {
+            file,
+            path,
+            dir: _dir,
+        } = TestFile::new(&[0u8; 8]);
+
+        let path_str = path.display().to_string();
+        let rpath = RString::new(&path_str);
+
+        // Object created successfully
+        let result: std::result::Result<Obj<MmapedFile>, Error> =
+            eval!("FastMmapedFileRs.new(path)", path = rpath);
+        assert!(result.is_ok());
+        let obj = result.unwrap();
+
+        let rs_self = &*obj;
+
+        let file_len = 1 << 32;
+        nix::unistd::ftruncate(file.as_raw_fd(), file_len).unwrap();
+
+        // We error when the file is externally modified.
+        assert!(rs_self.check_expand(obj, 512).is_err());
+
+        // We updated the object to use the new size.
+        assert_eq!(rs_self.capacity(), file_len as usize);
+    }
 }

data/lib/prometheus/client/configuration.rb

@@ -7,14 +7,13 @@ require 'tmpdir'
 module Prometheus
   module Client
     class Configuration
-      attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider, :rust_multiprocess_metrics
+      attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider
 
       def initialize
         @value_class = ::Prometheus::Client::MmapedValue
         @initial_mmap_file_size = ::Prometheus::Client::PageSize.page_size(fallback_page_size: 4096)
         @logger = Logger.new($stdout)
         @pid_provider = Process.method(:pid)
-        @rust_multiprocess_metrics = ENV.fetch('prometheus_rust_multiprocess_metrics', 'true') == 'true'
         @multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do
           Dir.mktmpdir("prometheus-mmap")
         end

data/lib/prometheus/client/formats/text.rb

@@ -1,6 +1,5 @@
 require 'prometheus/client/uses_value_type'
 require 'prometheus/client/helper/json_parser'
-require 'prometheus/client/helper/loader'
 require 'prometheus/client/helper/plain_file'
 require 'prometheus/client/helper/metrics_processing'
 require 'prometheus/client/helper/metrics_representation'
@@ -27,22 +26,12 @@ module Prometheus
             Helper::MetricsRepresentation.to_text(metrics)
           end
 
-          def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir, use_rust: true)
+          def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir)
             file_list = Dir.glob(File.join(path, '*.db')).sort
               .map {|f| Helper::PlainFile.new(f) }
               .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
 
-            if use_rust && Prometheus::Client::Helper::Loader.rust_impl_available?
               FastMmapedFileRs.to_metrics(file_list.to_a)
-            else
-              FastMmapedFile.to_metrics(file_list.to_a)
-            end
-          end
-
-          def rust_impl_available?
-            return @rust_available unless @rust_available.nil?
-
-            check_for_rust
           end
 
           private

data/lib/prometheus/client/helper/mmaped_file.rb

@@ -1,29 +1,18 @@
 require 'prometheus/client/helper/entry_parser'
 require 'prometheus/client/helper/file_locker'
-require 'prometheus/client/helper/loader'
 
 # load precompiled extension if available
 begin
   ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
-  require_relative "../../../#{ruby_version}/fast_mmaped_file"
+  require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
 rescue LoadError
-  require 'fast_mmaped_file'
+  require 'fast_mmaped_file_rs'
 end
 
 module Prometheus
   module Client
     module Helper
-      # We can't check `Prometheus::Client.configuration` as this creates a circular dependency
-      if (ENV.fetch('prometheus_rust_mmaped_file', 'true') == "true" &&
-          Prometheus::Client::Helper::Loader.rust_impl_available?)
-        class MmapedFile < FastMmapedFileRs
-        end
-      else
-        class MmapedFile < FastMmapedFile
-        end
-      end
-
-      class MmapedFile
+      class MmapedFile < FastMmapedFileRs
         include EntryParser
 
         attr_reader :filepath, :size

data/lib/prometheus/client/rack/exporter.rb

@@ -62,10 +62,8 @@ module Prometheus
         end
 
         def respond_with(format)
-          rust_enabled = Prometheus::Client.configuration.rust_multiprocess_metrics
-
           response = if Prometheus::Client.configuration.value_class.multiprocess
-                       format.marshal_multiprocess(use_rust: rust_enabled)
+                       format.marshal_multiprocess
                      else
                        format.marshal
                      end

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '0.28.1'.freeze
+    VERSION = '1.0.0'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 0.28.1
+  version: 1.0.0
 platform: x86_64-linux
 authors:
 - Tobias Schmidt
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-05 00:00:00.000000000 Z
+date: 2023-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -129,23 +129,6 @@ extra_rdoc_files: []
 files:
 - ".tool-versions"
 - README.md
-- ext/fast_mmaped_file/extconf.rb
-- ext/fast_mmaped_file/fast_mmaped_file.c
-- ext/fast_mmaped_file/file_format.c
-- ext/fast_mmaped_file/file_format.h
-- ext/fast_mmaped_file/file_parsing.c
-- ext/fast_mmaped_file/file_parsing.h
-- ext/fast_mmaped_file/file_reading.c
-- ext/fast_mmaped_file/file_reading.h
-- ext/fast_mmaped_file/globals.h
-- ext/fast_mmaped_file/mmap.c
-- ext/fast_mmaped_file/mmap.h
-- ext/fast_mmaped_file/rendering.c
-- ext/fast_mmaped_file/rendering.h
-- ext/fast_mmaped_file/utils.c
-- ext/fast_mmaped_file/utils.h
-- ext/fast_mmaped_file/value_access.c
-- ext/fast_mmaped_file/value_access.h
 - ext/fast_mmaped_file_rs/.cargo/config.toml
 - ext/fast_mmaped_file_rs/Cargo.lock
 - ext/fast_mmaped_file_rs/Cargo.toml
@@ -163,13 +146,9 @@ files:
 - ext/fast_mmaped_file_rs/src/raw_entry.rs
 - ext/fast_mmaped_file_rs/src/testhelper.rs
 - ext/fast_mmaped_file_rs/src/util.rs
-- lib/2.7/fast_mmaped_file.so
 - lib/2.7/fast_mmaped_file_rs.so
-- lib/3.0/fast_mmaped_file.so
 - lib/3.0/fast_mmaped_file_rs.so
-- lib/3.1/fast_mmaped_file.so
 - lib/3.1/fast_mmaped_file_rs.so
-- lib/3.2/fast_mmaped_file.so
 - lib/3.2/fast_mmaped_file_rs.so
 - lib/prometheus.rb
 - lib/prometheus/client.rb
@@ -180,7 +159,6 @@ files:
 - lib/prometheus/client/helper/entry_parser.rb
 - lib/prometheus/client/helper/file_locker.rb
 - lib/prometheus/client/helper/json_parser.rb
-- lib/prometheus/client/helper/loader.rb
 - lib/prometheus/client/helper/metrics_processing.rb
 - lib/prometheus/client/helper/metrics_representation.rb
 - lib/prometheus/client/helper/mmaped_file.rb
@@ -201,26 +179,6 @@ files:
 - lib/prometheus/client/support/unicorn.rb
 - lib/prometheus/client/uses_value_type.rb
 - lib/prometheus/client/version.rb
-- vendor/c/hashmap/.gitignore
-- vendor/c/hashmap/LICENSE
-- vendor/c/hashmap/README.md
-- vendor/c/hashmap/_config.yml
-- vendor/c/hashmap/src/hashmap.c
-- vendor/c/hashmap/src/hashmap.h
-- vendor/c/hashmap/test/Makefile
-- vendor/c/hashmap/test/hashmap_test.c
-- vendor/c/jsmn/.travis.yml
-- vendor/c/jsmn/LICENSE
-- vendor/c/jsmn/Makefile
-- vendor/c/jsmn/README.md
-- vendor/c/jsmn/example/jsondump.c
-- vendor/c/jsmn/example/simple.c
-- vendor/c/jsmn/jsmn.c
-- vendor/c/jsmn/jsmn.h
-- vendor/c/jsmn/library.json
-- vendor/c/jsmn/test/test.h
-- vendor/c/jsmn/test/tests.c
-- vendor/c/jsmn/test/testutil.h
 homepage: https://gitlab.com/gitlab-org/prometheus-client-mmap
 licenses:
 - Apache-2.0
@@ -246,6 +204,6 @@ requirements: []
 rubygems_version: 3.4.4
 signing_key: 
 specification_version: 4
-summary: A suite of instrumentation metric primitivesthat can be exposed through a
-  web services interface.
+summary: A suite of instrumentation metric primitives that can be exposed through
+  a web services interface.
 test_files: []

data/ext/fast_mmaped_file/extconf.rb

@@ -1,30 +0,0 @@
-require 'mkmf'
-require 'fileutils'
-
-$CFLAGS << ' -std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra'
-
-if enable_config('fail-on-warning')
-  $CFLAGS << ' -Werror'
-end
-
-if enable_config('debug')
-  $CFLAGS << ' -O0 -g'
-end
-
-if enable_config('address-sanitizer')
-  $CFLAGS << ' -O -fsanitize=address -fno-omit-frame-pointer -g'
-end
-
-CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
-
-cwd = File.expand_path(File.dirname(__FILE__))
-vendor_dir = File.join(cwd, '../../vendor/c')
-src_dir = File.join(cwd, '../../ext/fast_mmaped_file')
-
-src_files = %W[#{vendor_dir}/jsmn/jsmn.c #{vendor_dir}/hashmap/src/hashmap.c]
-FileUtils.cp(src_files, src_dir)
-
-$INCFLAGS << " -I#{vendor_dir}/jsmn -I#{vendor_dir}/hashmap/src"
-
-dir_config('fast_mmaped_file')
-create_makefile('fast_mmaped_file')