vinted-prometheus-client-mmap 1.5.0-x86_64-linux

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e813eec5d2233337d34cd3f19c79e68a9b38b63e45607b3dd0929fbcf84e353f
+  data.tar.gz: 979ebaef61c83ecf0e6eeb85c85581e3869c269e7868d3194d6d6faab2c83bf4
+SHA512:
+  metadata.gz: 1df8af4b3368826fde07874b89b14648a66f193a5c5a945bcda4fc6e9588500ddf800866ce2af04c573474b2c8aa5b1ee275674ba45dabc82539345b42f4ff10
+  data.tar.gz: 51c47ecccd1d1f0afb6c4543837c43657f203dc8b391b3ff4ed57a947499a6a7e98375be644eb5cd1033f2fc901f38cc4b663944ffb2a76c2ad8cbcd856bc27d

data/README.md

@@ -0,0 +1,5 @@
+# vinted-prometheus-client-mmap
+
+### ! This gem is part of the Engineering Hackathon, not to be used in production
+
+Forked prometheus-client-mmap gem

data/ext/fast_mmaped_file_rs/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "fast_mmaped_file_rs"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+prost = "0.12"
+prost-types = "0.12"
+hashbrown = "0.14"
+varint-rs = "2.2.0"
+itertools = "0.11.0"
+libc = "0.2"
+magnus = { version = "0.6", features = ["rb-sys"] }
+memmap2 = "0.9"
+# v0.26 cannot be built on CentOS 7 https://github.com/nix-rust/nix/issues/1972
+nix = { version = "0.25", features = ["mman"] } # mman used for MsFlags
+rb-sys = { version = "0.9", features = ["stable-api-compiled-fallback"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0", features = ["raw_value"] }
+smallvec = { version = "1.10", features = ["serde"] }
+thiserror = "1.0"
+
+[dev-dependencies]
+bstr = "1.9"
+indoc = "2.0"
+# We need the `embed` feature to run tests, but this triggers failures when building as a Gem.
+magnus = { version = "0.6", features = ["rb-sys","embed"] }
+rand = "0.8"
+sha2 = "0.10"
+tempfile = "3.9"
+
+[build-dependencies]
+rb-sys-env = "0.1"
+prost-build = "0.12"
+
+[lib]
+# Integration tests won't work if crate is only `cdylib`.
+crate-type = ["cdylib","lib"]

data/ext/fast_mmaped_file_rs/README.md

@@ -0,0 +1,52 @@
+# Testing
+
+## Running Tests
+
+Use [cargo nextest](https://nexte.st/) to execute the Rust unit tests.
+
+```sh
+$ cargo nextest run
+```
+
+## Why not use 'cargo test'?
+
+We need to embed Ruby into the test binary to access Ruby types. This requires
+us to run `magnus::embed::init()` no more than once before calling Ruby.
+See [the magnus docs](https://docs.rs/magnus/latest/magnus/embed/fn.init.html)
+for more details.
+
+If we try to create separate `#[test]` functions that call `init()` these will
+conflict, as Cargo runs tests in parallel using a single process with separate
+threads. Running `cargo test` will result in errors like:
+
+```
+---- file_info::test::with_ruby stdout ----
+thread 'file_info::test::with_ruby' panicked at 'Ruby already initialized'
+```
+
+The simplest workaround for this is to avoid using `cargo test` to run unit
+tests. [nextest](https://nexte.st/) is an alternate test harness that runs each
+test as its own process, enabling each test to intitialize Ruby without
+conflict.
+
+## 'symbol not found' errors when running tests
+
+If you see errors like the following when running tests:
+
+```
+Caused by:
+  for `fast_mmaped_file_rs`, command `/Users/myuser/prometheus-client-mmap/ext/fast_mmaped_file_rs/target/debug/deps/fast_mmaped_file_rs-c81ccc96a6484e04 --list --format terse` exited with signal 6 (SIGABRT)
+--- stdout:
+
+--- stderr:
+dyld[17861]: symbol not found in flat namespace '_rb_cArray'
+```
+
+Clearing the build cache will resolve the problem.
+
+```sh
+$ cargo clean
+```
+
+This is probably due to separate features being used with `magnus` in
+development builds.

data/ext/fast_mmaped_file_rs/build.rs

@@ -0,0 +1,7 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _ = rb_sys_env::activate()?;
+    prost_build::compile_protos(&["src/metrics.proto"], &["src/"])
+        .expect("failed compile protobufs");
+
+    Ok(())
+}

data/ext/fast_mmaped_file_rs/extconf.rb

@@ -0,0 +1,28 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+if find_executable('rustc')
+  create_rust_makefile("fast_mmaped_file_rs") do |r|
+    r.auto_install_rust_toolchain = false
+
+    if enable_config('fail-on-warning')
+      r.extra_rustflags = ["-Dwarnings"]
+    end
+
+    if enable_config('debug')
+      r.profile = :dev
+    end
+
+    if enable_config('address-sanitizer')
+      r.extra_rustflags = ["-Zsanitizer=address"]
+    end
+
+    # `rb_sys/mkmf` passes all arguments after `--` directly to `cargo rustc`.
+    # We use this awful hack to keep compatibility with existing flags used by
+    # the C implementation.
+    trimmed_argv = ARGV.take_while { |arg| arg != "--" }
+    ARGV = trimmed_argv
+  end
+else
+  raise 'rustc not found. prometheus-client-mmap now requires Rust.'
+end

data/ext/fast_mmaped_file_rs/src/error.rs

@@ -0,0 +1,174 @@
+use magnus::{exception, Ruby};
+use std::any;
+use std::fmt::Display;
+use std::io;
+use std::path::Path;
+use thiserror::Error;
+
+use crate::util;
+use crate::PROM_EPARSING_ERROR;
+
+/// A lightweight representation of Ruby ExceptionClasses.
+#[derive(PartialEq, Eq, Clone, Copy, Debug)]
+pub enum RubyError {
+    Arg,
+    Encoding,
+    Frozen,
+    Index,
+    Io,
+    NoMem,
+    PromParsing,
+    Runtime,
+    Type,
+}
+
+impl From<RubyError> for magnus::ExceptionClass {
+    fn from(err: RubyError) -> magnus::ExceptionClass {
+        match err {
+            RubyError::Arg => exception::arg_error(),
+            RubyError::Encoding => exception::encoding_error(),
+            RubyError::Frozen => exception::frozen_error(),
+            RubyError::Index => exception::index_error(),
+            RubyError::Io => exception::io_error(),
+            RubyError::NoMem => exception::no_mem_error(),
+            RubyError::Runtime => exception::runtime_error(),
+            RubyError::PromParsing => {
+                // UNWRAP: this will panic if called outside of a Ruby thread.
+                let ruby = Ruby::get().unwrap();
+                ruby.get_inner(&PROM_EPARSING_ERROR)
+            }
+            RubyError::Type => exception::type_error(),
+        }
+    }
+}
+
+/// Errors returned internally within the crate. Methods called directly by Ruby return
+/// `magnus::error::Error` as do functions that interact heavily with Ruby. This can be
+/// converted into a `magnus::error::Error` at the boundary between Rust and Ruby.
+#[derive(PartialEq, Eq, Error, Debug)]
+pub enum MmapError {
+    /// A read or write was made while another thread had mutable access to the mmap.
+    #[error("read/write operation attempted while mmap was being written to")]
+    ConcurrentAccess,
+    /// An error message used to exactly match the messages returned by the C
+    /// implementation.
+    #[error("{0}")]
+    Legacy(String, RubyError),
+    /// A String had invalid UTF-8 sequences.
+    #[error("{0}")]
+    Encoding(String),
+    /// A failed attempt to cast an integer from one type to another.
+    #[error("failed to cast {object_name} {value} from {from} to {to}")]
+    FailedCast {
+        from: &'static str,
+        to: &'static str,
+        value: String,
+        object_name: String,
+    },
+    /// The mmap was frozen when a mutable operation was attempted.
+    #[error("mmap")]
+    Frozen,
+    /// An io operation failed.
+    #[error("failed to {operation} path '{path}': {err}")]
+    Io {
+        operation: String,
+        path: String,
+        err: String,
+    },
+    #[error("string length gt {}", i32::MAX)]
+    KeyLength,
+    /// Failed to allocate memory.
+    #[error("Couldn't allocate for {0} memory")]
+    OutOfMemory(usize),
+    /// A memory operation fell outside of the containers bounds.
+    #[error("offset {index} out of bounds of len {len}")]
+    OutOfBounds { index: String, len: String },
+    /// A numeric operation overflowed.
+    #[error("overflow when {op} {value} and {added} of type {ty}")]
+    Overflow {
+        value: String,
+        added: String,
+        op: String,
+        ty: &'static str,
+    },
+    /// A miscellaneous error.
+    #[error("{0}")]
+    Other(String),
+    /// A failure when parsing a `.db` file containing Prometheus metrics.
+    #[error("{0}")]
+    PromParsing(String),
+    /// No mmap open.
+    #[error("unmapped file")]
+    UnmappedFile,
+    /// A custom error message with `strerror(3)` appended.
+    #[error("{0}")]
+    WithErrno(String),
+}
+
+impl MmapError {
+    pub fn legacy<T: Into<String>>(msg: T, ruby_err: RubyError) -> Self {
+        MmapError::Legacy(msg.into(), ruby_err)
+    }
+
+    pub fn failed_cast<T: Display, U>(value: T, object_name: &str) -> Self {
+        MmapError::FailedCast {
+            from: any::type_name::<T>(),
+            to: any::type_name::<U>(),
+            value: value.to_string(),
+            object_name: object_name.to_string(),
+        }
+    }
+    pub fn io(operation: &str, path: &Path, err: io::Error) -> Self {
+        MmapError::Io {
+            operation: operation.to_string(),
+            path: path.display().to_string(),
+            err: err.to_string(),
+        }
+    }
+
+    pub fn overflowed<T: Display>(value: T, added: T, op: &str) -> Self {
+        MmapError::Overflow {
+            value: value.to_string(),
+            added: added.to_string(),
+            op: op.to_string(),
+            ty: any::type_name::<T>(),
+        }
+    }
+
+    pub fn out_of_bounds<T: Display>(index: T, len: T) -> Self {      
+        MmapError::OutOfBounds {
+            index: index.to_string(),
+            len: len.to_string(),
+        }
+    }
+
+    pub fn with_errno<T: Into<String>>(msg: T) -> Self {
+        let strerror = util::strerror(util::errno());
+        MmapError::WithErrno(format!("{}: ({strerror})", msg.into()))
+    }
+
+    pub fn ruby_err(&self) -> RubyError {
+        match self {
+            MmapError::ConcurrentAccess => RubyError::Arg,
+            MmapError::Legacy(_, e) => *e,
+            MmapError::Encoding(_) => RubyError::Encoding,
+            MmapError::Io { .. } => RubyError::Io,
+            MmapError::FailedCast { .. } => RubyError::Arg,
+            MmapError::Frozen => RubyError::Frozen,
+            MmapError::KeyLength => RubyError::Arg,
+            MmapError::Overflow { .. } => RubyError::Arg,
+            MmapError::OutOfBounds { .. } => RubyError::Index,
+            MmapError::OutOfMemory { .. } => RubyError::NoMem,
+            MmapError::Other(_) => RubyError::Arg,
+            MmapError::PromParsing(_) => RubyError::PromParsing,
+            MmapError::UnmappedFile => RubyError::Io,
+            MmapError::WithErrno(_) => RubyError::Io,
+        }
+    }
+}
+
+impl From<MmapError> for magnus::error::Error {
+    fn from(err: MmapError) -> magnus::error::Error {
+        magnus::error::Error::new(err.ruby_err().into(), err.to_string())
+    }
+}

data/ext/fast_mmaped_file_rs/src/exemplars.rs

@@ -0,0 +1,25 @@
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Exemplar {
+    // Labels (set of label names/values). Only 1 for now.
+    // Value -> f64.
+    // Timestamp -> uint64.
+    // We have to cap the maximum size of strings.
+    // From the spec:
+    // The combined length of the label names and values of an Exemplar's LabelSet MUST NOT exceed 128 UTF-8 character code points. 
+    // 4 bytes max per code point.
+    // So, we need to allocate 128*4 = 512 bytes for the label names and values.
+    pub label_name: String,
+    
+    pub label_value: String,
+    pub value: f64,
+    pub timestamp: u128,
+}
+
+use serde::{Deserialize, Serialize};
+
+use crate::size_of;
+
+pub const EXEMPLAR_ENTRY_MAX_SIZE_BYTES:usize = 512 + size_of::<f64>() + size_of::<u64>();
+
+// Key -> use the old one.
+// Value -> allocate EXEMPLAR_ENTRY_MAX_SIZE_BYTES. If it exceeds this, we need to return an error. Use JSON.