prometheus-client-mmap 0.25.0 → 1.1.1

data/ext/fast_mmaped_file_rs/Cargo.toml

@@ -6,26 +6,26 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-hashbrown = "0.13"
+hashbrown = "0.14"
 libc = "0.2"
-magnus = { git = "https://github.com/matsadler/magnus", branch = "main", features = ["rb-sys"] }
-memmap2 = "0.5"
+magnus = { version = "0.6", features = ["rb-sys"] }
+memmap2 = "0.9"
 # v0.26 cannot be built on CentOS 7 https://github.com/nix-rust/nix/issues/1972
 nix = { version = "0.25", features = ["mman"] } # mman used for MsFlags
-rb-sys = "0.9"
+rb-sys = { version = "0.9", features = ["stable-api-compiled-fallback"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["raw_value"] }
 smallvec = { version = "1.10", features = ["serde"] }
 thiserror = "1.0"
 
 [dev-dependencies]
-bstr = "1.4"
+bstr = "1.9"
 indoc = "2.0"
 # We need the `embed` feature to run tests, but this triggers failures when building as a Gem.
-magnus = { git = "https://github.com/matsadler/magnus", branch = "main", features = ["rb-sys","embed"] }
+magnus = { version = "0.6", features = ["rb-sys","embed"] }
 rand = "0.8"
 sha2 = "0.10"
-tempfile = "3.5"
+tempfile = "3.9"
 
 [build-dependencies]
 rb-sys-env = "0.1"

data/ext/fast_mmaped_file_rs/extconf.rb

@@ -24,7 +24,5 @@ if find_executable('rustc')
     ARGV = trimmed_argv
   end
 else
-  puts 'rustc not found, skipping Rust extension.'
-
-  File.write('Makefile', dummy_makefile($srcdir).join(''))
+  raise 'rustc not found. prometheus-client-mmap now requires Rust.'
 end

data/ext/fast_mmaped_file_rs/src/file_info.rs

@@ -91,6 +91,11 @@ impl FileInfo {
 
         match self.file.read_to_end(buf) {
             Ok(n) if n == self.len => Ok(()),
+            // A worker may expand the file between our `stat` and `read`, no harm done.
+            Ok(n) if n > self.len => {
+                self.len = n;
+                Ok(())
+            }
             Ok(_) => Err(MmapError::io(
                 "read",
                 &self.path,
@@ -106,6 +111,8 @@ mod test {
     use magnus::{eval, RArray, Symbol};
     use rand::{thread_rng, Rng};
     use sha2::{Digest, Sha256};
+    use std::fs;
+    use std::io::Write;
 
     use super::*;
     use crate::testhelper::TestFile;
@@ -165,7 +172,7 @@ mod test {
 
         let mut info = FileInfo {
             file,
-            path,
+            path: path.clone(),
             len: buf.len(),
             multiprocess_mode: Symbol::new("puma"),
             type_: Symbol::new("max"),
@@ -187,4 +194,47 @@ mod test {
 
         assert_eq!(in_hash, out_hash, "content hashes");
     }
+
+    #[test]
+    fn test_read_from_file_resized() {
+        let _cleanup = unsafe { magnus::embed::init() };
+        let ruby = magnus::Ruby::get().unwrap();
+        crate::init(&ruby).unwrap();
+
+        const BUF_LEN: usize = 1 << 14; // 16KiB
+
+        // Create a buffer with random data.
+        let mut buf = vec![0u8; BUF_LEN];
+        thread_rng().fill(buf.as_mut_slice());
+
+        let TestFile {
+            file,
+            path,
+            dir: _dir,
+        } = TestFile::new(&buf);
+
+        let mut info = FileInfo {
+            file,
+            path: path.clone(),
+            len: buf.len(),
+            multiprocess_mode: Symbol::new("puma"),
+            type_: Symbol::new("max"),
+            pid: "worker-0_0".to_string(),
+        };
+
+        let mut resized_file = fs::OpenOptions::new()
+            .write(true)
+            .append(true)
+            .open(path)
+            .unwrap();
+
+        // Write data to file after it has been `stat`ed in the
+        // constructor.
+        resized_file.write_all(&[1; 1024]).unwrap();
+
+        let mut out_buf = Vec::new();
+        info.read_from_file(&mut out_buf).unwrap();
+
+        assert_eq!(BUF_LEN + 1024, info.len, "resized file updated len");
+    }
 }

data/ext/fast_mmaped_file_rs/src/mmap/inner.rs

@@ -224,16 +224,6 @@ impl InnerMmap {
         }
     }
 
-    /// Truncate the mmapped file to the end of the metrics data.
-    pub fn truncate_file(&mut self) -> Result<()> {
-        // CAST: no-op on 64-bit, widening on 32-bit.
-        let trunc_len = self.len as u64;
-
-        self.file
-            .set_len(trunc_len)
-            .map_err(|e| MmapError::legacy(format!("truncate: {e}"), RubyError::Type))
-    }
-
     /// Load the `used` header containing the size of the metrics data written.
     pub fn load_used(&self) -> Result<u32> {
         match read_u32(self.map.as_ref(), 0) {

data/ext/fast_mmaped_file_rs/src/mmap.rs

@@ -225,9 +225,6 @@ impl MmapedFile {
         let rs_self = &*rb_self;
 
         rs_self.inner_mut(|inner| {
-            // truncate file to actual used size
-            inner.truncate_file()?;
-
             // We are about to release the backing mmap for Ruby's String
             // objects. If Ruby attempts to read from them the program will
             // segfault. We update the length of all Strings to zero so Ruby
@@ -507,14 +504,14 @@ impl MmapedFile {
         match file.seek(SeekFrom::Start(len - 1)) {
             Ok(_) => {}
             Err(_) => {
-                return Err(MmapError::WithErrno(format!("Can't lseek {}", len - 1)));
+                return Err(MmapError::with_errno(format!("Can't lseek {}", len - 1)));
             }
         }
 
         match file.write(&[0x0]) {
             Ok(1) => {}
             _ => {
-                return Err(MmapError::WithErrno(format!(
+                return Err(MmapError::with_errno(format!(
                     "Can't extend {}",
                     path.display()
                 )));

data/lib/prometheus/client/configuration.rb

@@ -7,14 +7,13 @@ require 'tmpdir'
 module Prometheus
   module Client
     class Configuration
-      attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider, :rust_multiprocess_metrics
+      attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider
 
       def initialize
         @value_class = ::Prometheus::Client::MmapedValue
         @initial_mmap_file_size = ::Prometheus::Client::PageSize.page_size(fallback_page_size: 4096)
         @logger = Logger.new($stdout)
         @pid_provider = Process.method(:pid)
-        @rust_multiprocess_metrics = ENV.fetch('prometheus_rust_multiprocess_metrics', nil) == 'true'
         @multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do
           Dir.mktmpdir("prometheus-mmap")
         end

data/lib/prometheus/client/formats/text.rb

@@ -1,6 +1,5 @@
 require 'prometheus/client/uses_value_type'
 require 'prometheus/client/helper/json_parser'
-require 'prometheus/client/helper/loader'
 require 'prometheus/client/helper/plain_file'
 require 'prometheus/client/helper/metrics_processing'
 require 'prometheus/client/helper/metrics_representation'
@@ -27,22 +26,12 @@ module Prometheus
             Helper::MetricsRepresentation.to_text(metrics)
           end
 
-          def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir, use_rust: false)
+          def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir)
             file_list = Dir.glob(File.join(path, '*.db')).sort
               .map {|f| Helper::PlainFile.new(f) }
               .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
 
-            if use_rust && Prometheus::Client::Helper::Loader.rust_impl_available?
               FastMmapedFileRs.to_metrics(file_list.to_a)
-            else
-              FastMmapedFile.to_metrics(file_list.to_a)
-            end
-          end
-
-          def rust_impl_available?
-            return @rust_available unless @rust_available.nil?
-
-            check_for_rust
           end
 
           private

data/lib/prometheus/client/helper/mmaped_file.rb

@@ -1,29 +1,18 @@
 require 'prometheus/client/helper/entry_parser'
 require 'prometheus/client/helper/file_locker'
-require 'prometheus/client/helper/loader'
 
 # load precompiled extension if available
 begin
   ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
-  require_relative "../../../#{ruby_version}/fast_mmaped_file"
+  require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
 rescue LoadError
-  require 'fast_mmaped_file'
+  require 'fast_mmaped_file_rs'
 end
 
 module Prometheus
   module Client
     module Helper
-      # We can't check `Prometheus::Client.configuration` as this creates a circular dependency
-      if (ENV.fetch('prometheus_rust_mmaped_file', nil) == "true" &&
-          Prometheus::Client::Helper::Loader.rust_impl_available?)
-        class MmapedFile < FastMmapedFileRs
-        end
-      else
-        class MmapedFile < FastMmapedFile
-        end
-      end
-
-      class MmapedFile
+      class MmapedFile < FastMmapedFileRs
         include EntryParser
 
         attr_reader :filepath, :size

data/lib/prometheus/client/label_set_validator.rb

@@ -5,8 +5,7 @@ module Prometheus
     # LabelSetValidator ensures that all used label sets comply with the
     # Prometheus specification.
     class LabelSetValidator
-      # TODO: we might allow setting :instance in the future
-      RESERVED_LABELS = [:job, :instance].freeze
+      RESERVED_LABELS = [].freeze
 
       class LabelSetError < StandardError; end
       class InvalidLabelSetError < LabelSetError; end

data/lib/prometheus/client/rack/exporter.rb

@@ -62,10 +62,8 @@ module Prometheus
         end
 
         def respond_with(format)
-          rust_enabled = Prometheus::Client.configuration.rust_multiprocess_metrics
-
           response = if Prometheus::Client.configuration.value_class.multiprocess
-                       format.marshal_multiprocess(use_rust: rust_enabled)
+                       format.marshal_multiprocess
                      else
                        format.marshal
                      end

data/lib/prometheus/client/support/puma.rb

@@ -0,0 +1,44 @@
+module Prometheus
+  module Client
+    module Support
+      module Puma
+        extend self
+
+        def worker_pid_provider
+          wid = worker_id
+          if wid = worker_id
+            wid
+          else
+            "process_id_#{Process.pid}"
+          end
+        end
+
+        private
+
+        def object_based_worker_id
+          return unless defined?(::Puma::Cluster::Worker)
+
+          workers = ObjectSpace.each_object(::Puma::Cluster::Worker)
+          return if workers.nil?
+
+          workers_first = workers.first
+          workers_first.index unless workers_first.nil?
+        end
+
+        def program_name
+          $PROGRAM_NAME
+        end
+
+        def worker_id
+          if matchdata = program_name.match(/puma.*cluster worker ([0-9]+):/)
+            "puma_#{matchdata[1]}"
+          elsif object_worker_id = object_based_worker_id
+            "puma_#{object_worker_id}"
+          elsif program_name.include?('puma')
+            'puma_master'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '0.25.0'.freeze
+    VERSION = '1.1.1'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 0.25.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Tobias Schmidt
@@ -11,7 +11,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-15 00:00:00.000000000 Z
+date: 2024-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -19,14 +19,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.86
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.86
 - !ruby/object:Gem::Dependency
   name: fuzzbert
   requirement: !ruby/object:Gem::Requirement
@@ -95,14 +95,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
@@ -125,29 +125,11 @@ email:
 - wchandler@gitlab.com
 executables: []
 extensions:
-- ext/fast_mmaped_file/extconf.rb
 - ext/fast_mmaped_file_rs/extconf.rb
 extra_rdoc_files: []
 files:
 - ".tool-versions"
 - README.md
-- ext/fast_mmaped_file/extconf.rb
-- ext/fast_mmaped_file/fast_mmaped_file.c
-- ext/fast_mmaped_file/file_format.c
-- ext/fast_mmaped_file/file_format.h
-- ext/fast_mmaped_file/file_parsing.c
-- ext/fast_mmaped_file/file_parsing.h
-- ext/fast_mmaped_file/file_reading.c
-- ext/fast_mmaped_file/file_reading.h
-- ext/fast_mmaped_file/globals.h
-- ext/fast_mmaped_file/mmap.c
-- ext/fast_mmaped_file/mmap.h
-- ext/fast_mmaped_file/rendering.c
-- ext/fast_mmaped_file/rendering.h
-- ext/fast_mmaped_file/utils.c
-- ext/fast_mmaped_file/utils.h
-- ext/fast_mmaped_file/value_access.c
-- ext/fast_mmaped_file/value_access.h
 - ext/fast_mmaped_file_rs/.cargo/config.toml
 - ext/fast_mmaped_file_rs/Cargo.lock
 - ext/fast_mmaped_file_rs/Cargo.toml
@@ -174,7 +156,6 @@ files:
 - lib/prometheus/client/helper/entry_parser.rb
 - lib/prometheus/client/helper/file_locker.rb
 - lib/prometheus/client/helper/json_parser.rb
-- lib/prometheus/client/helper/loader.rb
 - lib/prometheus/client/helper/metrics_processing.rb
 - lib/prometheus/client/helper/metrics_representation.rb
 - lib/prometheus/client/helper/mmaped_file.rb
@@ -191,29 +172,10 @@ files:
 - lib/prometheus/client/registry.rb
 - lib/prometheus/client/simple_value.rb
 - lib/prometheus/client/summary.rb
+- lib/prometheus/client/support/puma.rb
 - lib/prometheus/client/support/unicorn.rb
 - lib/prometheus/client/uses_value_type.rb
 - lib/prometheus/client/version.rb
-- vendor/c/hashmap/.gitignore
-- vendor/c/hashmap/LICENSE
-- vendor/c/hashmap/README.md
-- vendor/c/hashmap/_config.yml
-- vendor/c/hashmap/src/hashmap.c
-- vendor/c/hashmap/src/hashmap.h
-- vendor/c/hashmap/test/Makefile
-- vendor/c/hashmap/test/hashmap_test.c
-- vendor/c/jsmn/.travis.yml
-- vendor/c/jsmn/LICENSE
-- vendor/c/jsmn/Makefile
-- vendor/c/jsmn/README.md
-- vendor/c/jsmn/example/jsondump.c
-- vendor/c/jsmn/example/simple.c
-- vendor/c/jsmn/jsmn.c
-- vendor/c/jsmn/jsmn.h
-- vendor/c/jsmn/library.json
-- vendor/c/jsmn/test/test.h
-- vendor/c/jsmn/test/tests.c
-- vendor/c/jsmn/test/testutil.h
 homepage: https://gitlab.com/gitlab-org/prometheus-client-mmap
 licenses:
 - Apache-2.0
@@ -236,6 +198,6 @@ requirements: []
 rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
-summary: A suite of instrumentation metric primitivesthat can be exposed through a
-  web services interface.
+summary: A suite of instrumentation metric primitives that can be exposed through
+  a web services interface.
 test_files: []

data/ext/fast_mmaped_file/extconf.rb

@@ -1,30 +0,0 @@
-require 'mkmf'
-require 'fileutils'
-
-$CFLAGS << ' -std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra'
-
-if enable_config('fail-on-warning')
-  $CFLAGS << ' -Werror'
-end
-
-if enable_config('debug')
-  $CFLAGS << ' -O0 -g'
-end
-
-if enable_config('address-sanitizer')
-  $CFLAGS << ' -O -fsanitize=address -fno-omit-frame-pointer -g'
-end
-
-CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
-
-cwd = File.expand_path(File.dirname(__FILE__))
-vendor_dir = File.join(cwd, '../../vendor/c')
-src_dir = File.join(cwd, '../../ext/fast_mmaped_file')
-
-src_files = %W[#{vendor_dir}/jsmn/jsmn.c #{vendor_dir}/hashmap/src/hashmap.c]
-FileUtils.cp(src_files, src_dir)
-
-$INCFLAGS << " -I#{vendor_dir}/jsmn -I#{vendor_dir}/hashmap/src"
-
-dir_config('fast_mmaped_file')
-create_makefile('fast_mmaped_file')

data/ext/fast_mmaped_file/fast_mmaped_file.c

@@ -1,122 +0,0 @@
-#include <errno.h>
-#include <hashmap.h>
-#include <jsmn.h>
-#include <ruby.h>
-#include <ruby/intern.h>
-#include <sys/mman.h>
-
-#include "file_parsing.h"
-#include "file_reading.h"
-#include "globals.h"
-#include "mmap.h"
-#include "rendering.h"
-#include "utils.h"
-#include "value_access.h"
-
-VALUE MMAPED_FILE = Qnil;
-
-ID sym_min;
-ID sym_max;
-ID sym_livesum;
-ID sym_gauge;
-ID sym_pid;
-ID sym_samples;
-
-VALUE prom_eParsingError;
-
-int aggregate_files(struct hashmap *map, VALUE list_of_files) {
-    buffer_t reading_buffer;
-    memset(&reading_buffer, 0, sizeof(buffer_t));
-
-    for (int i = 0; i < RARRAY_LEN(list_of_files); i++) {
-        VALUE params = RARRAY_PTR(list_of_files)[i];
-        file_t file;
-
-        if (!file_open_from_params(&file, params)) {
-            buffer_dispose(&reading_buffer);
-            return 0;
-        }
-
-        if (!read_from_file(&file, &reading_buffer)) {
-            buffer_dispose(&reading_buffer);
-            file_close(&file);
-            return 0;
-        }
-
-        if (!process_buffer(&file, &reading_buffer, map)) {
-            buffer_dispose(&reading_buffer);
-            file_close(&file);
-            return 0;
-        }
-
-        if (!file_close(&file)) {
-            buffer_dispose(&reading_buffer);
-            return 0;
-        }
-    }
-
-    buffer_dispose(&reading_buffer);
-    return 1;
-}
-
-VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list) {
-    struct hashmap map;
-    hashmap_setup(&map);
-
-    if (!aggregate_files(&map, file_list)) {  // all entries in map are now copies that need to be disposed
-        hashmap_destroy(&map);
-        raise_last_exception();
-        return Qnil;
-    }
-
-    entry_t **sorted_entries;
-
-    if (!sort_map_entries(&map, &sorted_entries)) {
-        hashmap_destroy(&map);
-
-        raise_last_exception();
-        return Qnil;
-    }
-
-    VALUE rv = rb_str_new("", 0);
-    if (!entries_to_string(rv, sorted_entries, hashmap_size(&map))) {
-        free(sorted_entries);
-        hashmap_destroy(&map);
-
-        raise_last_exception();
-        return Qnil;
-    }
-
-    RB_GC_GUARD(file_list);  // ensure file list is not GCed before this point
-    free(sorted_entries);
-    hashmap_destroy(&map);
-    return rv;
-}
-
-void Init_fast_mmaped_file() {
-    sym_gauge = rb_intern("gauge");
-    sym_min = rb_intern("min");
-    sym_max = rb_intern("max");
-    sym_livesum = rb_intern("livesum");
-    sym_pid = rb_intern("pid");
-    sym_samples = rb_intern("samples");
-
-    prom_eParsingError = rb_define_class("PrometheusParsingError", rb_eRuntimeError);
-
-    MMAPED_FILE = rb_define_class("FastMmapedFile", rb_cObject);
-    rb_define_const(MMAPED_FILE, "MAP_SHARED", INT2FIX(MAP_SHARED));
-
-    rb_define_singleton_method(MMAPED_FILE, "to_metrics", method_to_metrics, 1);
-
-    rb_define_alloc_func(MMAPED_FILE, mm_s_alloc);
-    rb_define_singleton_method(MMAPED_FILE, "new", mm_s_new, -1);
-    rb_define_method(MMAPED_FILE, "initialize", mm_init, 1);
-    rb_define_method(MMAPED_FILE, "slice", mm_aref_m, -1);
-    rb_define_method(MMAPED_FILE, "sync", mm_msync, -1);
-    rb_define_method(MMAPED_FILE, "munmap", mm_unmap, 0);
-
-    rb_define_method(MMAPED_FILE, "used", method_load_used, 0);
-    rb_define_method(MMAPED_FILE, "used=", method_save_used, 1);
-    rb_define_method(MMAPED_FILE, "fetch_entry", method_fetch_entry, 3);
-    rb_define_method(MMAPED_FILE, "upsert_entry", method_upsert_entry, 3);
-}

data/ext/fast_mmaped_file/file_format.c

@@ -1,5 +0,0 @@
-#include "file_format.h"
-
-inline uint32_t padding_length(uint32_t key_length) {
-    return 8 - (sizeof(uint32_t) + key_length) % 8;  // padding | 8 byte aligned
-}

data/ext/fast_mmaped_file/file_format.h

@@ -1,11 +0,0 @@
-#ifndef FILE_FORMAT_H
-#define FILE_FORMAT_H
-
-#include <stdint.h>
-
-#define START_POSITION 8
-#define INITIAL_SIZE (2 * sizeof(int32_t))
-
-uint32_t padding_length(uint32_t key_length);
-
-#endif