maxmind-db-rust 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 477024cc8d20f8e13fa22fa0e6d8a94554c91f16812e81ef2154ae5cc821739c
-  data.tar.gz: 4a5b94f8b821af36bed45f0b164ea1c5061c734a3827afc8f0ed059ec986c4b0
+  metadata.gz: 4da26d1a897424541b516fa63b4b5bc7a29c22f22b287f1da695464fe3f65dfb
+  data.tar.gz: f6bb57652b26da3cd5639f853be1ccd8cd6eac8c5d8f928d2220a41b76961ec2
 SHA512:
-  metadata.gz: ad9547ef7bf1d3371325d7fd519f78f0d01f302e5fecc4e5181d9a88e57aa1ebed4c51b1f6ae08491133aab05a63ad31516bae33051a78362308ec77ece1e204
-  data.tar.gz: f5be53fe4952c00ad82c3824a0b17aa7ae420a9580261686847f684ab8834fbbaa6d5b316405f6a70ee14fc9fd25717fcb367d49af6c5e9d29fb10ed33e2736b
+  metadata.gz: d8cdcdff702b4f7fbe6c01900f9d623038994ce1fbd7c4ce202689ee08432ff4e58c6cc269459e3fd2a9680bdee22f88ee10981a66d92679780bf2cce631246b
+  data.tar.gz: f5e1f10ea7ba02421a16d7eec91ff3435eaad367c9fd97d345105a769e7aec00fcc991e9e6ac85c682d5ec4f6ddfabf10cdc2cee55be1d29c05aa3ecb7700ead

data/CHANGELOG.md

@@ -5,6 +5,64 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.6.0] - 2026-07-19
+
+### Added
+
+- Added `Reader#verify` for opt-in comprehensive database integrity checks
+  using the verifier introduced by `maxminddb` 0.30.0.
+- Added iteration and shared-reader thread-scaling benchmarks, including a
+  cache-root retention check across thread churn.
+
+### Performance
+
+- Batched Ruby array and hash insertion while decoding MMDB records to reduce
+  protected Ruby C API calls during full-record lookups.
+- Replaced per-thread string caches with one fixed-size process cache that uses
+  its Ruby root array directly, improving reuse while bounding retained memory.
+- Batched `get_many` and `get_many_path` result construction to reduce Ruby
+  array insertion overhead for large input collections.
+- Improved `IPAddr` lookup performance by converting its public integer
+  representation directly instead of allocating packed address strings.
+- Avoided allocating a temporary candidate vector on parsed-path cache hits.
+
+### Changed
+
+- Upgraded the `maxminddb` crate to 0.30.0, improving selective path decoding
+  and hardening iteration and verification of corrupt databases.
+- Documented selective lookup and input-type performance guidance, and
+  clarified that thread-safe lookups remain serialized by MRI's global VM lock.
+- Documented the file-lifecycle safety contract for memory-mapped readers and
+  the requirement to atomically replace database paths during updates.
+- Simplified reader ownership by removing redundant outer `Arc` wrappers and a
+  duplicate closed flag while preserving atomic close semantics.
+- Removed the redundant extension-level Cargo lockfile; workspace builds use
+  the root lockfile as the single dependency lock.
+- Moved MMDB-to-Ruby decoding and string-cache logic into a focused Rust module.
+- Moved metadata storage and Ruby method registration into a focused Rust
+  module.
+- Moved parsed-path ownership, hashing, matching, and conversion into a focused
+  Rust module.
+- Moved reader behavior and Ruby method registration into a focused Rust module,
+  leaving the crate root responsible only for extension initialization.
+- Moved extension namespace and cache-root setup into a focused initialization
+  module.
+- Alternated baseline and candidate subprocesses between benchmark samples to
+  reduce order and thermal bias in git-ref comparisons.
+- Added fixed-record and prebuilt-`IPAddr` benchmark cases for cache-hot and
+  input-conversion performance comparisons.
+- Extended allocation benchmarking to cover selective batch lookups.
+
+### Fixed
+
+- Made decoded MMDB strings consistently frozen regardless of whether their
+  length qualifies for the bounded string cache.
+- Decoded MMDB UTF-8 strings and map keys directly into Ruby strings from raw
+  bytes, avoiding unchecked Rust strings while preserving Ruby's behavior for
+  invalid UTF-8 in corrupt databases.
+- Propagated Ruby hash insertion errors from `Metadata#description` instead of
+  silently returning a partial description.
+
 ## [0.5.0] - 2026-06-14
 
 ### Added
@@ -156,3 +214,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - MODE_FILE support (use MODE_MMAP instead)
 - File descriptor support in constructor
+
+[0.6.0]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.5.0...v0.6.0
+[0.5.0]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.4.0...v0.5.0
+[0.4.0]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.3.0...v0.4.0
+[0.3.0]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.2.1...v0.3.0
+[0.2.1]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.2.0...v0.2.1
+[0.2.0]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.1.4...v0.2.0
+[0.1.4]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.1.3...v0.1.4
+[0.1.3]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.1.2...v0.1.3
+[0.1.2]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.1.1...v0.1.2
+[0.1.1]: https://github.com/oschwald/maxmind-db-rust-ruby/compare/v0.1.0...v0.1.1
+[0.1.0]: https://github.com/oschwald/maxmind-db-rust-ruby/releases/tag/v0.1.0

data/README.md

@@ -102,6 +102,9 @@ iso_codes = reader.get_many_path(ips, ['country', 'iso_code'])
 reader.close
 ```
 
+Decoded MMDB strings, including hash keys, are frozen. Call `dup` before
+modifying a decoded string.
+
 ### Using IPAddr Objects
 
 ```ruby
@@ -153,6 +156,11 @@ reader = MaxMind::DB::Rust::Reader.new(
 )
 ```
 
+`MODE_AUTO`, `MODE_FILE`, and `MODE_MMAP` require the underlying file not to be
+modified or truncated while the reader is alive. To refresh a database, write
+the replacement to a new file and atomically rename it over the old path. Use
+`MODE_MEMORY` if the file lifecycle cannot be controlled this way.
+
 ### Accessing Metadata
 
 ```ruby
@@ -290,6 +298,18 @@ Get metadata about the database.
 
 **Returns:** `MaxMind::DB::Rust::Metadata` instance
 
+#### `verify()`
+
+Perform a comprehensive validation of the database metadata, search tree, data
+section separator, and referenced data records. Verification traverses the
+entire database and may use memory proportional to the number of distinct
+referenced values, so it is intended for explicit integrity checks rather than
+the lookup hot path.
+
+**Returns:** `true` when the database passes verification
+
+**Raises:** `MaxMind::DB::Rust::InvalidDatabaseError` when verification fails
+
 #### `close()`
 
 Close the database and release resources.
@@ -365,8 +385,16 @@ Lookup performance depends on hardware, Ruby version, database, and workload.
 
 - In this project’s random-lookup benchmarks, this gem is consistently faster than the official Ruby implementation.
 - `MODE_MMAP` and `MODE_MEMORY` both perform well; which is faster can vary by environment.
+- Prefer `get_path` or `get_many_path` when only a small part of a record is
+  needed. Selective decoding avoids constructing the rest of the Ruby object
+  graph and can be substantially faster than a full-record lookup.
+- Textual IP address strings use the fastest input path. `IPAddr` objects are
+  supported but require Ruby method calls to extract their address family and
+  integer value.
 - For current, reproducible numbers on your own data and Ruby version, run `benchmark/compare_lookups.rb` against your database.
-- Safe for concurrent lookups across threads.
+- Readers are safe to share across Ruby threads. Ruby-facing lookup and decode
+  work currently runs under MRI's global VM lock, so additional Ruby threads do
+  not make one process perform lookups in parallel.
 
 ## Development
 

data/ext/maxmind_db_rust/Cargo.toml

@@ -15,7 +15,7 @@ crate-type = ["cdylib"]
 arc-swap = "1.9"
 ipnetwork = "0.21"
 magnus = "0.8"
-maxminddb = { version = "0.28.1", features = ["unsafe-str-decode"] }
+maxminddb = "0.30.0"
 memmap2 = "0.9"
 rustc-hash = "2.1"
 serde = "1.0"

data/ext/maxmind_db_rust/src/decoder.rs

@@ -0,0 +1,324 @@
+use crate::{
+    rust_module, STRING_CACHE_MAX, STRING_CACHE_MAX_LEN, STRING_CACHE_MIN_LEN,
+    STRING_CACHE_ROOTS_CONST,
+};
+use ::maxminddb as maxminddb_crate;
+use magnus::{prelude::*, IntoValue, RArray, RString, Value};
+use rustc_hash::FxHasher;
+use serde::de::{self, Deserialize, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor};
+use std::{
+    cell::OnceCell,
+    fmt,
+    hash::{Hash, Hasher},
+};
+
+thread_local! {
+    static STRING_CACHE_ROOTS: OnceCell<RArray> = const { OnceCell::new() };
+}
+
+#[inline]
+fn global_string_cache_roots(ruby: &magnus::Ruby) -> RArray {
+    let value = rust_module(ruby)
+        .const_get::<_, Value>(STRING_CACHE_ROOTS_CONST)
+        .expect("string cache roots constant should exist");
+    RArray::from_value(value).expect("string cache roots constant should be an array")
+}
+
+#[inline]
+fn string_cache_roots(ruby: &magnus::Ruby) -> RArray {
+    STRING_CACHE_ROOTS.with(|roots| *roots.get_or_init(|| global_string_cache_roots(ruby)))
+}
+
+#[inline]
+fn cached_utf8_string(ruby: &magnus::Ruby, value: &[u8]) -> Value {
+    if !(STRING_CACHE_MIN_LEN..=STRING_CACHE_MAX_LEN).contains(&value.len()) {
+        let string = ruby.enc_str_new(value, ruby.utf8_encoding());
+        string.freeze();
+        return string.into_value_with(ruby);
+    }
+
+    let mut hasher = FxHasher::default();
+    value.hash(&mut hasher);
+    let hash = hasher.finish();
+    let slot = (hash as usize) & (STRING_CACHE_MAX - 1);
+
+    let roots = string_cache_roots(ruby);
+    let cached = roots
+        .entry::<Value>(slot as isize)
+        .expect("string cache roots lookup should succeed");
+    if let Some(cached) = RString::from_value(cached) {
+        // SAFETY: the bytes are compared immediately while the globally rooted
+        // frozen Ruby string remains alive and cannot be mutated.
+        if unsafe { cached.as_slice() } == value {
+            return cached.into_value_with(ruby);
+        }
+    }
+
+    let string = ruby.enc_str_new(value, ruby.utf8_encoding());
+    string.freeze();
+    let cached = string.as_value();
+    roots
+        .store(slot as isize, cached)
+        .expect("string cache roots update should succeed");
+    cached
+}
+
+/// Wrapper that owns the Ruby value produced by deserializing a MaxMind record.
+#[derive(Clone)]
+pub(crate) struct RubyDecodedValue {
+    value: Value,
+}
+
+impl RubyDecodedValue {
+    #[inline]
+    fn new(value: Value) -> Self {
+        Self { value }
+    }
+
+    #[inline]
+    pub(crate) fn into_value(self) -> Value {
+        self.value
+    }
+}
+
+impl<'de> Deserialize<'de> for RubyDecodedValue {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let ruby = magnus::Ruby::get().expect("Ruby VM should be available in deserializer");
+        RubyValueSeed { ruby: &ruby }.deserialize(deserializer)
+    }
+}
+
+struct RubyValueSeed<'ruby> {
+    ruby: &'ruby magnus::Ruby,
+}
+
+impl<'ruby, 'de> DeserializeSeed<'de> for RubyValueSeed<'ruby> {
+    type Value = RubyDecodedValue;
+
+    fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        maxminddb_crate::deserialize_any_with_raw_strings(
+            deserializer,
+            RubyValueVisitor { ruby: self.ruby },
+        )
+    }
+}
+
+struct RubyValueVisitor<'ruby> {
+    ruby: &'ruby magnus::Ruby,
+}
+
+impl<'de, 'ruby> Visitor<'de> for RubyValueVisitor<'ruby> {
+    type Value = RubyDecodedValue;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("any valid MaxMind DB value")
+    }
+
+    fn visit_bool<E>(self, value: bool) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(value.into_value_with(self.ruby)))
+    }
+
+    fn visit_i32<E>(self, value: i32) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(value.into_value_with(self.ruby)))
+    }
+
+    fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value >= i32::MIN as i64 && value <= i32::MAX as i64 {
+            Ok(RubyDecodedValue::new(
+                (value as i32).into_value_with(self.ruby),
+            ))
+        } else {
+            Ok(RubyDecodedValue::new(value.into_value_with(self.ruby)))
+        }
+    }
+
+    fn visit_u16<E>(self, value: u16) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(value.into_value_with(self.ruby)))
+    }
+
+    fn visit_u32<E>(self, value: u32) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(value.into_value_with(self.ruby)))
+    }
+
+    fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(value.into_value_with(self.ruby)))
+    }
+
+    fn visit_u128<E>(self, value: u128) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(value.into_value_with(self.ruby)))
+    }
+
+    fn visit_f32<E>(self, value: f32) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(
+            (value as f64).into_value_with(self.ruby),
+        ))
+    }
+
+    fn visit_f64<E>(self, value: f64) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(value.into_value_with(self.ruby)))
+    }
+
+    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer
+            .deserialize_bytes(RubyUtf8StringVisitor { ruby: self.ruby })
+            .map(RubyDecodedValue::new)
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(cached_utf8_string(
+            self.ruby,
+            value.as_bytes(),
+        )))
+    }
+
+    fn visit_string<E>(self, value: String) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(cached_utf8_string(
+            self.ruby,
+            value.as_bytes(),
+        )))
+    }
+
+    fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(
+            self.ruby.str_from_slice(value).into_value_with(self.ruby),
+        ))
+    }
+
+    fn visit_byte_buf<E>(self, value: Vec<u8>) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(RubyDecodedValue::new(
+            self.ruby.str_from_slice(&value).into_value_with(self.ruby),
+        ))
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: SeqAccess<'de>,
+    {
+        let arr = match seq.size_hint() {
+            Some(cap) => self.ruby.ary_new_capa(cap),
+            None => self.ruby.ary_new(),
+        };
+        let mut buffer = [self.ruby.qnil().as_value(); 128];
+        let mut buffer_len = 0;
+        while let Some(elem) = seq.next_element_seed(RubyValueSeed { ruby: self.ruby })? {
+            buffer[buffer_len] = elem.into_value();
+            buffer_len += 1;
+            if buffer_len == buffer.len() {
+                arr.cat(&buffer)
+                    .map_err(|e| de::Error::custom(e.to_string()))?;
+                buffer_len = 0;
+            }
+        }
+        arr.cat(&buffer[..buffer_len])
+            .map_err(|e| de::Error::custom(e.to_string()))?;
+        Ok(RubyDecodedValue::new(arr.into_value_with(self.ruby)))
+    }
+
+    fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+    where
+        A: MapAccess<'de>,
+    {
+        let hash = match map.size_hint() {
+            Some(cap) => self.ruby.hash_new_capa(cap),
+            None => self.ruby.hash_new(),
+        };
+        let mut buffer = [self.ruby.qnil().as_value(); 128];
+        let mut buffer_len = 0;
+        while let Some(key_val) = map.next_key_seed(RubyMapKeySeed { ruby: self.ruby })? {
+            let value = map.next_value_seed(RubyValueSeed { ruby: self.ruby })?;
+            buffer[buffer_len] = key_val;
+            buffer[buffer_len + 1] = value.into_value();
+            buffer_len += 2;
+            if buffer_len == buffer.len() {
+                hash.bulk_insert(&buffer)
+                    .map_err(|e| de::Error::custom(e.to_string()))?;
+                buffer_len = 0;
+            }
+        }
+        hash.bulk_insert(&buffer[..buffer_len])
+            .map_err(|e| de::Error::custom(e.to_string()))?;
+        Ok(RubyDecodedValue::new(hash.into_value_with(self.ruby)))
+    }
+}
+
+struct RubyUtf8StringVisitor<'ruby> {
+    ruby: &'ruby magnus::Ruby,
+}
+
+impl<'de, 'ruby> Visitor<'de> for RubyUtf8StringVisitor<'ruby> {
+    type Value = Value;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("MMDB UTF-8 string bytes")
+    }
+
+    fn visit_borrowed_bytes<E>(self, value: &'de [u8]) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(cached_utf8_string(self.ruby, value))
+    }
+}
+
+struct RubyMapKeySeed<'ruby> {
+    ruby: &'ruby magnus::Ruby,
+}
+
+impl<'ruby, 'de> DeserializeSeed<'de> for RubyMapKeySeed<'ruby> {
+    type Value = Value;
+
+    fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_identifier(RubyUtf8StringVisitor { ruby: self.ruby })
+    }
+}

data/ext/maxmind_db_rust/src/initialization.rs

@@ -0,0 +1,75 @@
+use crate::{metadata, reader, STRING_CACHE_MAX, STRING_CACHE_ROOTS_CONST};
+use magnus::{error::Error, prelude::*, RModule, Value};
+
+const MAP_KEY_ROOTS_CONST: &str = "__MAP_KEY_ROOTS__";
+
+pub(crate) fn initialize(ruby: &magnus::Ruby) -> Result<(), Error> {
+    // Define module hierarchy: MaxMind::DB::Rust
+    // Handle case where official maxmind-db gem may have already defined MaxMind::DB as a Class
+    let maxmind = ruby.define_module("MaxMind")?;
+
+    // Try to get or define DB - it might be a Class (official gem) or Module (ours)
+    let db_value = maxmind.const_get::<_, Value>("DB");
+    let rust = match db_value {
+        Ok(existing) if existing.is_kind_of(ruby.class_class()) => {
+            // MaxMind::DB exists as a Class (official gem loaded first)
+            // Reuse existing Rust constant if present to avoid replacing classes.
+            if let Ok(rust_value) = existing.funcall::<_, _, Value>("const_get", ("Rust", false)) {
+                RModule::from_value(rust_value).ok_or_else(|| {
+                    Error::new(
+                        ruby.exception_type_error(),
+                        "MaxMind::DB::Rust exists but is not a module",
+                    )
+                })?
+            } else {
+                // Define Rust module directly as a constant on the class.
+                let rust_value: Value = ruby.module_new().as_value();
+                let rust_mod = RModule::from_value(rust_value).ok_or_else(|| {
+                    Error::new(
+                        ruby.exception_type_error(),
+                        "Failed to create anonymous module for MaxMind::DB::Rust",
+                    )
+                })?;
+                let _ = existing.funcall::<_, _, Value>("const_set", ("Rust", rust_mod))?;
+                rust_mod
+            }
+        }
+        Ok(existing) => {
+            // MaxMind::DB exists as a Module (our gem loaded first)
+            let db_mod = RModule::from_value(existing).ok_or_else(|| {
+                Error::new(ruby.exception_type_error(), "MaxMind::DB is not a module")
+            })?;
+            db_mod.define_module("Rust")?
+        }
+        Err(_) => {
+            // MaxMind::DB doesn't exist, define it as a module
+            let db = maxmind.define_module("DB")?;
+            db.define_module("Rust")?
+        }
+    };
+
+    if rust
+        .const_get::<_, Value>(STRING_CACHE_ROOTS_CONST)
+        .is_err()
+    {
+        let roots = ruby.ary_new_capa(STRING_CACHE_MAX);
+        for _ in 0..STRING_CACHE_MAX {
+            roots.push(ruby.qnil().as_value())?;
+        }
+        rust.const_set(STRING_CACHE_ROOTS_CONST, roots)?;
+    }
+
+    if rust.const_get::<_, Value>(MAP_KEY_ROOTS_CONST).is_ok() {
+        let _ = rust.funcall::<_, _, Value>("send", ("remove_const", MAP_KEY_ROOTS_CONST))?;
+    }
+
+    // The extension can be loaded more than once from different paths.
+    // Reusing previously defined classes avoids typed-data incompatibilities.
+    if rust.const_get::<_, Value>("Reader").is_ok() {
+        return Ok(());
+    }
+
+    reader::define(ruby, rust)?;
+    metadata::define(ruby, rust)?;
+    Ok(())
+}