prometheus-client-mmap 0.23.0-x86_64-linux → 0.24.3-x86_64-linux

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7851e4bc5f33acf49fffc52ff51d7fb63061df60da9e383dc0cc15e4704917c2
4
- data.tar.gz: 00d412c3d379bcde0690f25f64609feed218541537bb198bbf1adbcbeecc72ce
3
+ metadata.gz: 96d81412cc07d51c07e2a4d3d63d24a0977f8f3218a469bace79a9d2abc929b6
4
+ data.tar.gz: 61c024688e436092d09d4b77214603e1c374be69e6b4b289d0d2997f10218c24
5
5
  SHA512:
6
- metadata.gz: fe5642626ed2434a3a279e4eed3b548baa0c4a97bde02961a6346fcd456217d10b2b65fe53cd16d13554b5af50cd3a3ecba4b86bd539ab6fa351ead690339d74
7
- data.tar.gz: 68c9990cd351879d489c09251118f03cecb1f4f3c2a0f47a8d6380061a2172c17a2377e6d3fa902b7002ee735f3363b1b7376811495f65758acfd88710eb3ec9
6
+ metadata.gz: a030d17f52d1530a30e8ff892632a00ff8cd10489ded1a7268eca58898dfd73c627f56f38bd8382066c70d9e58400719e37f1485790fcea723d377080512712c
7
+ data.tar.gz: 6a2bf596a09c5a7a255051f4c6945f2f325d6d0db03b0a9f537af50ade9b1b17356240628723401391b8eaa27ad66e2d9b7730c9c65b5a040049d19663b429b1
data/.tool-versions ADDED
@@ -0,0 +1 @@
1
+ rust 1.65.0
data/README.md CHANGED
@@ -200,6 +200,17 @@ Set `prometheus_multiproc_dir` environment variable to the path where you want m
200
200
  prometheus_multiproc_dir=/tmp
201
201
  ```
202
202
 
203
+ ### Multiprocess metrics via Rust extension
204
+
205
+ If the environment variable `prometheus_rust_multiprocess_metrics=true` is set or if the `rust_multiprocess_metrics`
206
+ configuration setting is `true` and the `fast_mmaped_file_rs` extension is available, it will be used to generate
207
+ multiprocess metrics. This should be significantly faster than the C extension.
208
+
209
+ ### Read and write metrics via Rust extension
210
+
211
+ If the environment variable `prometheus_rust_mmaped_file=true` is set then if the `fast_mmaped_file_rs`
212
+ extension is available it will be used to read and write metrics from the mmapped file.
213
+
203
214
  ## Pitfalls
204
215
 
205
216
  ### PID cardinality
@@ -9,7 +9,7 @@ use crate::util;
9
9
  use crate::PROM_EPARSING_ERROR;
10
10
 
11
11
  /// A lightweight representation of Ruby ExceptionClasses.
12
- #[derive(PartialEq, Clone, Copy, Debug)]
12
+ #[derive(PartialEq, Eq, Clone, Copy, Debug)]
13
13
  pub enum RubyError {
14
14
  Arg,
15
15
  Encoding,
@@ -45,7 +45,7 @@ impl From<RubyError> for magnus::ExceptionClass {
45
45
  /// Errors returned internally within the crate. Methods called directly by Ruby return
46
46
  /// `magnus::error::Error` as do functions that interact heavily with Ruby. This can be
47
47
  /// converted into a `magnus::error::Error` at the boundary between Rust and Ruby.
48
- #[derive(PartialEq, Error, Debug)]
48
+ #[derive(PartialEq, Eq, Error, Debug)]
49
49
  pub enum MmapError {
50
50
  /// A read or write was made while another thread had mutable access to the mmap.
51
51
  #[error("read/write operation attempted while mmap was being written to")]
@@ -1,4 +1,4 @@
1
- use magnus::{StaticSymbol, Symbol};
1
+ use magnus::Symbol;
2
2
  use std::fmt::Write;
3
3
  use std::str;
4
4
 
@@ -80,7 +80,7 @@ impl<'a> BorrowedData<'a> {
80
80
  #[derive(Clone, Debug)]
81
81
  pub struct EntryMetadata {
82
82
  pub multiprocess_mode: Symbol,
83
- pub type_: StaticSymbol,
83
+ pub type_: Symbol,
84
84
  pub value: f64,
85
85
  }
86
86
 
@@ -176,7 +176,7 @@ impl FileEntry {
176
176
  out.push_str(family_name);
177
177
  out.push(' ');
178
178
 
179
- out.push_str(self.meta.type_.name().expect("name was invalid UTF-8"));
179
+ out.push_str(&self.meta.type_.name().expect("name was invalid UTF-8"));
180
180
  out.push('\n');
181
181
  }
182
182
 
@@ -438,7 +438,7 @@ mod test {
438
438
  path,
439
439
  len: case.json.len(),
440
440
  multiprocess_mode: Symbol::new(case.multiprocess_mode),
441
- type_: StaticSymbol::new("gauge"),
441
+ type_: Symbol::new("gauge"),
442
442
  pid: pid.to_string(),
443
443
  };
444
444
  file_infos.push(info);
@@ -544,7 +544,7 @@ mod test {
544
544
  path,
545
545
  len: json.len(),
546
546
  multiprocess_mode: Symbol::new(case.multiprocess_mode),
547
- type_: StaticSymbol::new(case.metric_type),
547
+ type_: Symbol::new(case.metric_type),
548
548
  pid: "worker-1".to_string(),
549
549
  };
550
550
 
@@ -1,5 +1,5 @@
1
1
  use magnus::exception::*;
2
- use magnus::{Error, RString, StaticSymbol, Symbol, Value};
2
+ use magnus::{Error, RString, Symbol, Value};
3
3
  use std::ffi::OsString;
4
4
  use std::fs::File;
5
5
  use std::io::{self, Read, Seek};
@@ -18,7 +18,7 @@ pub struct FileInfo {
18
18
  pub path: PathBuf,
19
19
  pub len: usize,
20
20
  pub multiprocess_mode: Symbol,
21
- pub type_: StaticSymbol,
21
+ pub type_: Symbol,
22
22
  pub pid: String,
23
23
  }
24
24
 
@@ -59,7 +59,7 @@ impl FileInfo {
59
59
  let multiprocess_mode = Symbol::from_value(params[1])
60
60
  .ok_or_else(|| err!(arg_error(), "expected multiprocess_mode to be a symbol"))?;
61
61
 
62
- let type_ = StaticSymbol::from_value(params[2])
62
+ let type_ = Symbol::from_value(params[2])
63
63
  .ok_or_else(|| err!(arg_error(), "expected file type to be a symbol"))?;
64
64
 
65
65
  let pid = RString::from_value(params[3])
@@ -103,7 +103,7 @@ impl FileInfo {
103
103
 
104
104
  #[cfg(test)]
105
105
  mod test {
106
- use magnus::{eval, RArray, StaticSymbol, Symbol};
106
+ use magnus::{eval, RArray, Symbol};
107
107
  use rand::{thread_rng, Rng};
108
108
  use sha2::{Digest, Sha256};
109
109
 
@@ -168,7 +168,7 @@ mod test {
168
168
  path,
169
169
  len: buf.len(),
170
170
  multiprocess_mode: Symbol::new("puma"),
171
- type_: StaticSymbol::new("max"),
171
+ type_: Symbol::new("max"),
172
172
  pid: "worker-0_0".to_string(),
173
173
  };
174
174
 
@@ -166,7 +166,7 @@ impl EntryMap {
166
166
 
167
167
  #[cfg(test)]
168
168
  mod test {
169
- use magnus::{StaticSymbol, Symbol};
169
+ use magnus::Symbol;
170
170
  use std::mem;
171
171
 
172
172
  use super::*;
@@ -197,7 +197,7 @@ mod test {
197
197
  },
198
198
  meta: EntryMetadata {
199
199
  multiprocess_mode: Symbol::new("max"),
200
- type_: StaticSymbol::new("gauge"),
200
+ type_: Symbol::new("gauge"),
201
201
  value: 1.0,
202
202
  },
203
203
  },
@@ -208,7 +208,7 @@ mod test {
208
208
  },
209
209
  meta: EntryMetadata {
210
210
  multiprocess_mode: Symbol::new("max"),
211
- type_: StaticSymbol::new("gauge"),
211
+ type_: Symbol::new("gauge"),
212
212
  value: 1.0,
213
213
  },
214
214
  },
@@ -219,7 +219,7 @@ mod test {
219
219
  },
220
220
  meta: EntryMetadata {
221
221
  multiprocess_mode: Symbol::new("max"),
222
- type_: StaticSymbol::new("gauge"),
222
+ type_: Symbol::new("gauge"),
223
223
  value: 1.0,
224
224
  },
225
225
  },
@@ -230,7 +230,7 @@ mod test {
230
230
  },
231
231
  meta: EntryMetadata {
232
232
  multiprocess_mode: Symbol::new("max"),
233
- type_: StaticSymbol::new("gauge"),
233
+ type_: Symbol::new("gauge"),
234
234
  value: 1.0,
235
235
  },
236
236
  },
@@ -241,7 +241,7 @@ mod test {
241
241
  },
242
242
  meta: EntryMetadata {
243
243
  multiprocess_mode: Symbol::new("all"),
244
- type_: StaticSymbol::new("gauge"),
244
+ type_: Symbol::new("gauge"),
245
245
  value: 1.0,
246
246
  },
247
247
  },
@@ -252,7 +252,7 @@ mod test {
252
252
  },
253
253
  meta: EntryMetadata {
254
254
  multiprocess_mode: Symbol::new("all"),
255
- type_: StaticSymbol::new("gauge"),
255
+ type_: Symbol::new("gauge"),
256
256
  value: 1.0,
257
257
  },
258
258
  },
@@ -293,7 +293,7 @@ mod test {
293
293
  },
294
294
  meta: EntryMetadata {
295
295
  multiprocess_mode: Symbol::new("all"),
296
- type_: StaticSymbol::new("gauge"),
296
+ type_: Symbol::new("gauge"),
297
297
  value: 1.0,
298
298
  },
299
299
  };
@@ -305,7 +305,7 @@ mod test {
305
305
  },
306
306
  meta: EntryMetadata {
307
307
  multiprocess_mode: Symbol::new("all"),
308
- type_: StaticSymbol::new("gauge"),
308
+ type_: Symbol::new("gauge"),
309
309
  value: 5.0,
310
310
  },
311
311
  };
@@ -317,7 +317,7 @@ mod test {
317
317
  },
318
318
  meta: EntryMetadata {
319
319
  multiprocess_mode: Symbol::new("all"),
320
- type_: StaticSymbol::new("gauge"),
320
+ type_: Symbol::new("gauge"),
321
321
  value: 100.0,
322
322
  },
323
323
  };
@@ -329,7 +329,7 @@ mod test {
329
329
  },
330
330
  meta: EntryMetadata {
331
331
  multiprocess_mode: Symbol::new("all"),
332
- type_: StaticSymbol::new("gauge"),
332
+ type_: Symbol::new("gauge"),
333
333
  value: 1.0,
334
334
  },
335
335
  };
@@ -460,7 +460,7 @@ mod test {
460
460
  path,
461
461
  len: case.json.len(),
462
462
  multiprocess_mode: Symbol::new("max"),
463
- type_: StaticSymbol::new("gauge"),
463
+ type_: Symbol::new("gauge"),
464
464
  pid: "worker-1".to_string(),
465
465
  };
466
466
 
@@ -204,6 +204,12 @@ impl InnerMmap {
204
204
  self.map.as_ptr()
205
205
  }
206
206
 
207
+ /// Returns a mutable raw pointer to the mmap.
208
+ /// For use in updating RString internals which requires a mutable pointer.
209
+ pub fn as_mut_ptr(&self) -> *mut u8 {
210
+ self.map.as_ptr().cast_mut()
211
+ }
212
+
207
213
  /// Perform an msync(2) on the mmap, flushing all changes written
208
214
  /// to disk. The sync may optionally be performed asynchronously.
209
215
  pub fn flush(&mut self, f_async: bool) -> Result<()> {
@@ -3,7 +3,8 @@ use magnus::prelude::*;
3
3
  use magnus::rb_sys::{AsRawValue, FromRawValue};
4
4
  use magnus::typed_data::Obj;
5
5
  use magnus::value::Fixnum;
6
- use magnus::{block::Proc, eval, scan_args, Error, Integer, RArray, RClass, RHash, RString, Value};
6
+ use magnus::{eval, scan_args, Error, Integer, RArray, RClass, RHash, RString, Value};
7
+ use nix::libc::{c_char, c_long, c_ulong};
7
8
  use rb_sys::rb_str_new_static;
8
9
  use std::fs::File;
9
10
  use std::io::{prelude::*, SeekFrom};
@@ -24,6 +25,11 @@ use inner::InnerMmap;
24
25
 
25
26
  mod inner;
26
27
 
28
+ /// The Ruby `STR_NOEMBED` flag, aka `FL_USER1`.
29
+ const STR_NOEMBED: c_ulong = 1 << (13);
30
+ /// The Ruby `STR_SHARED` flag, aka `FL_USER2`.
31
+ const STR_SHARED: c_ulong = 1 << (14);
32
+
27
33
  /// A Rust struct wrapped in a Ruby object, providing access to a memory-mapped
28
34
  /// file used to store, update, and read out Prometheus metrics.
29
35
  ///
@@ -150,13 +156,28 @@ impl MmapedFile {
150
156
  ///
151
157
  /// return a substring of <em>lenght</em> characters from <em>start</em>
152
158
  pub fn slice(rb_self: Obj<Self>, args: &[Value]) -> magnus::error::Result<RString> {
153
- // The C implemenation would trigger a GC cycle via `rb_gc_force_recycle`
159
+ // The C implementation would trigger a GC cycle via `rb_gc_force_recycle`
154
160
  // if the `MM_PROTECT` flag is set, but in practice this is never used.
155
161
  // We omit this logic, particularly because `rb_gc_force_recycle` is a
156
162
  // no-op as of Ruby 3.1.
163
+ let rs_self = &*rb_self;
164
+
165
+ let str = rs_self.str(rb_self)?;
166
+ rs_self._slice(rb_self, str, args)
167
+ }
157
168
 
158
- let str = (*rb_self).str(rb_self)?;
159
- let res = str.funcall("[]", args);
169
+ fn _slice(
170
+ &self,
171
+ rb_self: Obj<Self>,
172
+ str: RString,
173
+ args: &[Value],
174
+ ) -> magnus::error::Result<RString> {
175
+ let substr: RString = str.funcall("[]", args)?;
176
+
177
+ // Track shared child strings which use the same backing storage.
178
+ if Self::rb_string_is_shared(substr) {
179
+ (*rb_self).track_rstring(rb_self, substr)?;
180
+ }
160
181
 
161
182
  // The C implementation does this, perhaps to validate that the len we
162
183
  // provided is actually being used.
@@ -165,7 +186,7 @@ impl MmapedFile {
165
186
  Ok(())
166
187
  })?;
167
188
 
168
- res
189
+ Ok(substr)
169
190
  }
170
191
 
171
192
  /// Document-method: msync
@@ -222,7 +243,8 @@ impl MmapedFile {
222
243
  })?;
223
244
 
224
245
  // Update each String object to be zero-length.
225
- rs_self.update_weak_map(rb_self)?;
246
+ let cap = util::cast_chk::<_, c_long>(rs_self.capacity(), "capacity")?;
247
+ rs_self.update_weak_map(rb_self, rs_self.as_mut_ptr(), cap)?;
226
248
 
227
249
  // Remove the `InnerMmap` from the `RwLock`. This will drop
228
250
  // end of this function, unmapping and closing the file.
@@ -336,44 +358,44 @@ impl MmapedFile {
336
358
 
337
359
  // SAFETY: This is safe so long as the data provided to Ruby meets its
338
360
  // requirements. When unmapping the file this will no longer be the
339
- // case, see the comment on `munmap` how we handle this.
361
+ // case, see the comment on `munmap` for how we handle this.
340
362
  Ok(unsafe { rb_str_new_static(ptr as _, len as _) })
341
363
  })?;
342
364
 
365
+ // SAFETY: We know that rb_str_new_static returns a VALUE.
343
366
  let val = unsafe { Value::from_raw(val_id) };
344
367
 
345
368
  // UNWRAP: We created this value as a string above.
346
369
  let str = RString::from_value(val).unwrap();
347
370
 
348
- let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
371
+ // Freeze the root string so it can't be mutated out from under any
372
+ // substrings created. This object is never exposed to callers.
373
+ str.freeze();
349
374
 
350
- // Use the string's Id as the key in the `WeakMap`.
351
- let key = str.as_raw();
352
- let _: Value = tracker.funcall("[]=", (key, str))?;
375
+ // Track the RString in our `WeakMap` so we can update its address if
376
+ // we re-mmap the backing file.
377
+ (*rb_self).track_rstring(rb_self, str)?;
353
378
 
354
379
  Ok(str)
355
380
  }
356
381
 
357
382
  /// If we reallocate, any live Ruby strings provided by the `str()` method
358
- /// will be will be invalidated. We need to iterate over them using and
359
- /// update their heap pointers to the newly allocated memory region.
360
- fn update_weak_map(&self, rb_self: Obj<Self>) -> magnus::error::Result<()> {
383
+ /// will be invalidated. We need to iterate over them using and update their
384
+ /// heap pointers to the newly allocated memory region.
385
+ fn update_weak_map(
386
+ &self,
387
+ rb_self: Obj<Self>,
388
+ old_ptr: *const c_char,
389
+ old_cap: c_long,
390
+ ) -> magnus::error::Result<()> {
361
391
  let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
362
392
 
363
- let (new_ptr, new_len) = (*rb_self).inner(|inner| {
364
- // Pointers are not `Send`, convert it to usize to allow capture in closure.
365
- let new_ptr = inner.as_ptr() as usize;
366
- let new_len = util::cast_chk::<_, i64>(inner.len(), "mmap len")?;
393
+ let new_len = self.inner(|inner| util::cast_chk::<_, c_long>(inner.len(), "mmap len"))?;
367
394
 
368
- Ok((new_ptr, new_len))
369
- })?;
370
-
371
- // Allocate a block with a closure that updates the string details.
372
- let block = Proc::from_fn(move |args, _block| {
373
- let val = args
374
- .get(0)
375
- .ok_or_else(|| err!(arg_error(), "no argument received from `each_value`"))?;
376
- let str = RString::from_value(*val)
395
+ // Iterate over the values of the `WeakMap`.
396
+ for val in tracker.enumeratorize("each_value", ()) {
397
+ let rb_string = val?;
398
+ let str = RString::from_value(rb_string)
377
399
  .ok_or_else(|| err!(arg_error(), "weakmap value was not a string"))?;
378
400
 
379
401
  // SAFETY: We're messing with Ruby's internals here, YOLO.
@@ -382,18 +404,42 @@ impl MmapedFile {
382
404
  // which provides access to its internals.
383
405
  let mut raw_str = Self::rb_string_internal(str);
384
406
 
407
+ // Shared string have their own `ptr` and `len` values, but `aux`
408
+ // is the id of the parent string so the GC can track this
409
+ // dependency. The `ptr` will always be an offset from the base
410
+ // address of the mmap, and `len` will be the length of the mmap
411
+ // less the offset from the base.
412
+ if Self::rb_string_is_shared(str) && new_len > 0 {
413
+ // Calculate how far into the original mmap the shared string
414
+ // started and update to the equivalent address in the new
415
+ // one.
416
+ let substr_ptr = raw_str.as_ref().as_.heap.ptr;
417
+ let offset = substr_ptr.offset_from(old_ptr);
418
+
419
+ raw_str.as_mut().as_.heap.ptr = self.as_mut_ptr().offset(offset);
420
+
421
+ let current_len = raw_str.as_ref().as_.heap.len;
422
+ let new_shared_len = old_cap + current_len;
423
+
424
+ raw_str.as_mut().as_.heap.len = new_shared_len;
425
+ continue;
426
+ }
427
+
385
428
  // Update the string to point to the new mmapped file.
386
429
  // We're matching the behavior of Ruby's `str_new_static` function.
387
430
  // See https://github.com/ruby/ruby/blob/e51014f9c05aa65cbf203442d37fef7c12390015/string.c#L1030-L1053
388
- raw_str.as_mut().as_.heap.ptr = new_ptr as _;
431
+ //
432
+ // We deliberately do _NOT_ increment the `capa` field of the
433
+ // string to match the new `len`. We were initially doing this,
434
+ // but consistently triggered GCs in the middle of updating the
435
+ // string pointers, causing a segfault.
436
+ //
437
+ // See https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/issues/45
438
+ raw_str.as_mut().as_.heap.ptr = self.as_mut_ptr();
389
439
  raw_str.as_mut().as_.heap.len = new_len;
390
- raw_str.as_mut().as_.heap.aux.capa = new_len;
391
440
  }
392
- Ok(())
393
- });
441
+ }
394
442
 
395
- // Execute the block.
396
- let _: Value = tracker.funcall_with_block("each_value", (), block)?;
397
443
  Ok(())
398
444
  }
399
445
 
@@ -406,8 +452,7 @@ impl MmapedFile {
406
452
 
407
453
  // We need the mmapped region to contain at least one byte beyond the
408
454
  // written data to create a NUL- terminated C string. Validate that
409
- // new length does not exactly match the length of the mmap or exceed
410
- // it.
455
+ // new length does not exactly match or exceed the length of the mmap.
411
456
  while self.capacity() <= used.add_chk(entry_len)? {
412
457
  self.expand_to_fit(rb_self, self.capacity().mul_chk(2)?)?;
413
458
  }
@@ -429,6 +474,9 @@ impl MmapedFile {
429
474
  }
430
475
 
431
476
  if new_cap != self.capacity() {
477
+ let old_ptr = self.as_mut_ptr();
478
+ let old_cap = util::cast_chk::<_, c_long>(self.capacity(), "capacity")?;
479
+
432
480
  // Drop the old mmap.
433
481
  let (mut file, path) = self.take_inner()?.munmap();
434
482
 
@@ -439,7 +487,7 @@ impl MmapedFile {
439
487
 
440
488
  self.insert_inner(new_inner)?;
441
489
 
442
- return self.update_weak_map(rb_self);
490
+ return self.update_weak_map(rb_self, old_ptr, old_cap);
443
491
  }
444
492
 
445
493
  Ok(())
@@ -476,6 +524,15 @@ impl MmapedFile {
476
524
  Ok(())
477
525
  }
478
526
 
527
+ fn track_rstring(&self, rb_self: Obj<Self>, str: RString) -> magnus::error::Result<()> {
528
+ let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
529
+
530
+ // Use the string's Id as the key in the `WeakMap`.
531
+ let key = str.as_raw();
532
+ let _: Value = tracker.funcall("[]=", (key, str))?;
533
+ Ok(())
534
+ }
535
+
479
536
  /// The total capacity of the underlying mmap.
480
537
  #[inline]
481
538
  fn capacity(&self) -> usize {
@@ -489,6 +546,13 @@ impl MmapedFile {
489
546
  .map_err(|e| e.into())
490
547
  }
491
548
 
549
+ fn as_mut_ptr(&self) -> *mut c_char {
550
+ // UNWRAP: This is actually infallible, but we need to
551
+ // wrap it in a `Result` for use with `inner()`.
552
+ self.inner(|inner| Ok(inner.as_mut_ptr() as *mut c_char))
553
+ .unwrap()
554
+ }
555
+
492
556
  /// Takes a closure with immutable access to InnerMmap. Will fail if the inner
493
557
  /// object has a mutable borrow or has been dropped.
494
558
  fn inner<F, T>(&self, func: F) -> Result<T>
@@ -544,9 +608,27 @@ impl MmapedFile {
544
608
  Ok(())
545
609
  }
546
610
 
611
+ /// Check if an RString is shared. Shared string use the same underlying
612
+ /// storage as their parent, taking an offset from the start. By default
613
+ /// they must run to the end of the parent string.
614
+ fn rb_string_is_shared(rb_str: RString) -> bool {
615
+ // SAFETY: We only hold a reference to the raw object for the duration
616
+ // of this function, and no Ruby code is called.
617
+ let flags = unsafe {
618
+ let raw_str = Self::rb_string_internal(rb_str);
619
+ raw_str.as_ref().basic.flags
620
+ };
621
+ let shared_flags = STR_SHARED | STR_NOEMBED;
622
+
623
+ flags & shared_flags == shared_flags
624
+ }
625
+
547
626
  /// Convert `magnus::RString` into the raw binding used by `rb_sys::RString`.
548
627
  /// We need this to manually change the pointer and length values for strings
549
628
  /// when moving the mmap to a new file.
629
+ ///
630
+ /// SAFETY: Calling Ruby code while the returned object is held may result
631
+ /// in it being mutated or dropped.
550
632
  unsafe fn rb_string_internal(rb_str: RString) -> NonNull<rb_sys::RString> {
551
633
  mem::transmute::<RString, NonNull<rb_sys::RString>>(rb_str)
552
634
  }
@@ -666,18 +748,101 @@ mod test {
666
748
  let ruby = magnus::Ruby::get().unwrap();
667
749
  crate::init(&ruby).unwrap();
668
750
 
751
+ fn assert_internals(
752
+ obj: Obj<MmapedFile>,
753
+ parent_id: c_ulong,
754
+ child_id: c_ulong,
755
+ unshared_id: c_ulong,
756
+ ) {
757
+ let rs_self = &*obj;
758
+ let tracker: Value = obj.ivar_get("@weak_obj_tracker").unwrap();
759
+
760
+ let mmap_ptr = rs_self.as_mut_ptr();
761
+ let mmap_len = rs_self.capacity();
762
+
763
+ let mut parent_checked = false;
764
+ let mut child_checked = false;
765
+
766
+ for val in tracker.enumeratorize("each_value", ()) {
767
+ let rb_string = val.unwrap();
768
+ let str = RString::from_value(rb_string).unwrap();
769
+
770
+ unsafe {
771
+ let raw_str = MmapedFile::rb_string_internal(str);
772
+ if str.as_raw() == child_id {
773
+ assert_eq!(parent_id, raw_str.as_ref().as_.heap.aux.shared);
774
+
775
+ let child_offset =
776
+ mmap_len as isize - raw_str.as_ref().as_.heap.len as isize;
777
+ assert_eq!(mmap_ptr.offset(child_offset), raw_str.as_ref().as_.heap.ptr);
778
+
779
+ child_checked = true;
780
+ } else if str.as_raw() == parent_id {
781
+ assert_eq!(parent_id, str.as_raw());
782
+
783
+ assert_eq!(mmap_ptr, raw_str.as_ref().as_.heap.ptr);
784
+ assert_eq!(mmap_len as c_long, raw_str.as_ref().as_.heap.len);
785
+ assert!(raw_str.as_ref().basic.flags & (STR_SHARED | STR_NOEMBED) > 0);
786
+ assert!(str.is_frozen());
787
+
788
+ parent_checked = true;
789
+ } else if str.as_raw() == unshared_id {
790
+ panic!("tracking unshared string");
791
+ } else {
792
+ panic!("unknown string");
793
+ }
794
+ }
795
+ }
796
+ assert!(parent_checked && child_checked);
797
+ }
798
+
669
799
  let obj = create_obj();
670
800
  let _ = populate_entries(&obj);
671
801
 
672
802
  let rs_self = &*obj;
673
803
 
674
- let value_range = Range::new(HEADER_SIZE, 24, false).unwrap().as_value();
675
- let value_slice = MmapedFile::slice(obj, &[value_range]).unwrap();
676
-
677
- rs_self.expand_to_fit(obj, rs_self.capacity() * 2).unwrap();
804
+ // Create a string containing the full mmap.
805
+ let parent_str = rs_self.str(obj).unwrap();
806
+ let parent_id = parent_str.as_raw();
807
+
808
+ // Ruby's shared strings are only created when they go to the end of
809
+ // original string.
810
+ let len = rs_self.inner(|inner| Ok(inner.len())).unwrap();
811
+ let shareable_range = Range::new(1, len - 1, false).unwrap().as_value();
812
+
813
+ // This string should re-use the parent's buffer with an offset and have
814
+ // the parent's id in `as.heap.aux.shared`
815
+ let child_str = rs_self._slice(obj, parent_str, &[shareable_range]).unwrap();
816
+ let child_id = child_str.as_raw();
817
+
818
+ // A range that does not reach the end of the parent will not be shared.
819
+ assert!(len > 4);
820
+ let unshareable_range = Range::new(0, 4, false).unwrap().as_value();
821
+
822
+ // This string should NOT be tracked, it should own its own buffer.
823
+ let unshared_str = rs_self
824
+ ._slice(obj, parent_str, &[unshareable_range])
825
+ .unwrap();
826
+ let unshared_id = unshared_str.as_raw();
827
+ assert!(!MmapedFile::rb_string_is_shared(unshared_str));
828
+
829
+ assert_internals(obj, parent_id, child_id, unshared_id);
830
+
831
+ let orig_ptr = rs_self.as_mut_ptr();
832
+ // Expand a bunch to ensure we remap
833
+ for _ in 0..16 {
834
+ rs_self.expand_to_fit(obj, rs_self.capacity() * 2).unwrap();
835
+ }
836
+ let new_ptr = rs_self.as_mut_ptr();
837
+ assert!(orig_ptr != new_ptr);
678
838
 
679
839
  // If we haven't updated the pointer to the newly remapped file this will segfault.
680
- let _: Value = eval!("puts str", str = value_slice).unwrap();
840
+ let _: Value = eval!("puts parent", parent = parent_str).unwrap();
841
+ let _: Value = eval!("puts child", child = child_str).unwrap();
842
+ let _: Value = eval!("puts unshared", unshared = unshared_str).unwrap();
843
+
844
+ // Confirm that tracked strings are still valid.
845
+ assert_internals(obj, parent_id, child_id, unshared_id);
681
846
  }
682
847
 
683
848
  #[test]
@@ -2,7 +2,7 @@ use smallvec::SmallVec;
2
2
  use std::str;
3
3
 
4
4
  /// String slices pointing to the fields of a borrowed `Entry`'s JSON data.
5
- #[derive(PartialEq, Debug)]
5
+ #[derive(PartialEq, Eq, Debug)]
6
6
  pub struct MetricText<'a> {
7
7
  pub family_name: &'a str,
8
8
  pub metric_name: &'a str,
@@ -10,14 +10,14 @@ pub struct MetricText<'a> {
10
10
  pub values: SmallVec<[&'a str; 4]>,
11
11
  }
12
12
 
13
- #[derive(PartialEq, Debug)]
13
+ #[derive(PartialEq, Eq, Debug)]
14
14
  struct MetricNames<'a> {
15
15
  label_json: &'a str,
16
16
  family_name: &'a str,
17
17
  metric_name: &'a str,
18
18
  }
19
19
 
20
- #[derive(PartialEq, Debug)]
20
+ #[derive(PartialEq, Eq, Debug)]
21
21
  struct MetricLabelVals<'a> {
22
22
  labels: SmallVec<[&'a str; 4]>,
23
23
  values: SmallVec<[&'a str; 4]>,
@@ -6,7 +6,7 @@ use crate::util::CheckedOps;
6
6
  use crate::Result;
7
7
 
8
8
  /// The logic to save a `MetricsEntry`, or parse one from a byte slice.
9
- #[derive(PartialEq, Clone, Debug)]
9
+ #[derive(PartialEq, Eq, Clone, Debug)]
10
10
  pub struct RawEntry<'a> {
11
11
  bytes: &'a [u8],
12
12
  encoded_len: usize,
@@ -73,7 +73,7 @@ pub fn entries_to_db(entries: &[&'static str], values: &[f64], header: Option<u3
73
73
  };
74
74
 
75
75
  out.extend(used.to_ne_bytes());
76
- out.extend(&[0x0u8; 4]); // Padding.
76
+ out.extend([0x0u8; 4]); // Padding.
77
77
  out.extend(entry_bytes);
78
78
 
79
79
  out
Binary file
Binary file
Binary file
Binary file
@@ -7,13 +7,14 @@ require 'tmpdir'
7
7
  module Prometheus
8
8
  module Client
9
9
  class Configuration
10
- attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider
10
+ attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider, :rust_multiprocess_metrics
11
11
 
12
12
  def initialize
13
13
  @value_class = ::Prometheus::Client::MmapedValue
14
14
  @initial_mmap_file_size = ::Prometheus::Client::PageSize.page_size(fallback_page_size: 4096)
15
15
  @logger = Logger.new($stdout)
16
16
  @pid_provider = Process.method(:pid)
17
+ @rust_multiprocess_metrics = ENV.fetch('prometheus_rust_multiprocess_metrics', nil) == 'true'
17
18
  @multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do
18
19
  Dir.mktmpdir("prometheus-mmap")
19
20
  end
@@ -1,5 +1,6 @@
1
1
  require 'prometheus/client/uses_value_type'
2
2
  require 'prometheus/client/helper/json_parser'
3
+ require 'prometheus/client/helper/loader'
3
4
  require 'prometheus/client/helper/plain_file'
4
5
  require 'prometheus/client/helper/metrics_processing'
5
6
  require 'prometheus/client/helper/metrics_representation'
@@ -31,7 +32,7 @@ module Prometheus
31
32
  .map {|f| Helper::PlainFile.new(f) }
32
33
  .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
33
34
 
34
- if use_rust && rust_impl_available?
35
+ if use_rust && Prometheus::Client::Helper::Loader.rust_impl_available?
35
36
  FastMmapedFileRs.to_metrics(file_list.to_a)
36
37
  else
37
38
  FastMmapedFile.to_metrics(file_list.to_a)
@@ -46,29 +47,6 @@ module Prometheus
46
47
 
47
48
  private
48
49
 
49
- def load_rust_extension
50
- begin
51
- ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
52
- require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
53
- rescue LoadError
54
- require 'fast_mmaped_file_rs'
55
- end
56
- end
57
-
58
- def check_for_rust
59
- # This will be evaluated on each invocation even with `||=` if
60
- # `@rust_available` if false. Running a `require` statement is slow,
61
- # so the `rust_impl_available?` method memoizes the result, external
62
- # callers can only trigger this method a single time.
63
- @rust_available = begin
64
- load_rust_extension
65
- true
66
- rescue LoadError
67
- Prometheus::Client.logger.info('FastMmapedFileRs unavailable')
68
- false
69
- end
70
- end
71
-
72
50
  def load_metrics(path)
73
51
  metrics = {}
74
52
  Dir.glob(File.join(path, '*.db')).sort.each do |f|
@@ -0,0 +1,40 @@
1
+ module Prometheus
2
+ module Client
3
+ module Helper
4
+ module Loader
5
+ class << self
6
+ def rust_impl_available?
7
+ return @rust_available unless @rust_available.nil?
8
+
9
+ check_for_rust
10
+ end
11
+
12
+ private
13
+
14
+ def load_rust_extension
15
+ begin
16
+ ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
17
+ require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
18
+ rescue LoadError
19
+ require 'fast_mmaped_file_rs'
20
+ end
21
+ end
22
+
23
+ def check_for_rust
24
+ # This will be evaluated on each invocation even with `||=` if
25
+ # `@rust_available` if false. Running a `require` statement is slow,
26
+ # so the `rust_impl_available?` method memoizes the result, external
27
+ # callers can only trigger this method a single time.
28
+ @rust_available = begin
29
+ load_rust_extension
30
+ true
31
+ rescue LoadError
32
+ Prometheus::Client.logger.info('FastMmapedFileRs unavailable')
33
+ false
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -1,5 +1,6 @@
1
1
  require 'prometheus/client/helper/entry_parser'
2
2
  require 'prometheus/client/helper/file_locker'
3
+ require 'prometheus/client/helper/loader'
3
4
 
4
5
  # load precompiled extension if available
5
6
  begin
@@ -12,7 +13,17 @@ end
12
13
  module Prometheus
13
14
  module Client
14
15
  module Helper
15
- class MmapedFile < FastMmapedFile
16
+ # We can't check `Prometheus::Client.configuration` as this creates a circular dependency
17
+ if (ENV.fetch('prometheus_rust_mmaped_file', nil) == "true" &&
18
+ Prometheus::Client::Helper::Loader.rust_impl_available?)
19
+ class MmapedFile < FastMmapedFileRs
20
+ end
21
+ else
22
+ class MmapedFile < FastMmapedFile
23
+ end
24
+ end
25
+
26
+ class MmapedFile
16
27
  include EntryParser
17
28
 
18
29
  attr_reader :filepath, :size
@@ -62,8 +62,10 @@ module Prometheus
62
62
  end
63
63
 
64
64
  def respond_with(format)
65
+ rust_enabled = Prometheus::Client.configuration.rust_multiprocess_metrics
66
+
65
67
  response = if Prometheus::Client.configuration.value_class.multiprocess
66
- format.marshal_multiprocess
68
+ format.marshal_multiprocess(use_rust: rust_enabled)
67
69
  else
68
70
  format.marshal
69
71
  end
@@ -1,5 +1,5 @@
1
1
  module Prometheus
2
2
  module Client
3
- VERSION = '0.23.0'.freeze
3
+ VERSION = '0.24.3'.freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: prometheus-client-mmap
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.23.0
4
+ version: 0.24.3
5
5
  platform: x86_64-linux
6
6
  authors:
7
7
  - Tobias Schmidt
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2023-05-01 00:00:00.000000000 Z
14
+ date: 2023-05-20 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: rb_sys
@@ -127,6 +127,7 @@ executables: []
127
127
  extensions: []
128
128
  extra_rdoc_files: []
129
129
  files:
130
+ - ".tool-versions"
130
131
  - README.md
131
132
  - ext/fast_mmaped_file/extconf.rb
132
133
  - ext/fast_mmaped_file/fast_mmaped_file.c
@@ -179,6 +180,7 @@ files:
179
180
  - lib/prometheus/client/helper/entry_parser.rb
180
181
  - lib/prometheus/client/helper/file_locker.rb
181
182
  - lib/prometheus/client/helper/json_parser.rb
183
+ - lib/prometheus/client/helper/loader.rb
182
184
  - lib/prometheus/client/helper/metrics_processing.rb
183
185
  - lib/prometheus/client/helper/metrics_representation.rb
184
186
  - lib/prometheus/client/helper/mmaped_file.rb