prometheus-client-mmap 0.23.1-x86_64-linux → 0.24.3-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +11 -0
- data/ext/fast_mmaped_file_rs/src/error.rs +2 -2
- data/ext/fast_mmaped_file_rs/src/file_entry.rs +5 -5
- data/ext/fast_mmaped_file_rs/src/file_info.rs +5 -5
- data/ext/fast_mmaped_file_rs/src/map.rs +12 -12
- data/ext/fast_mmaped_file_rs/src/mmap/inner.rs +6 -0
- data/ext/fast_mmaped_file_rs/src/mmap.rs +202 -37
- data/ext/fast_mmaped_file_rs/src/parser.rs +3 -3
- data/ext/fast_mmaped_file_rs/src/raw_entry.rs +1 -1
- data/ext/fast_mmaped_file_rs/src/testhelper.rs +1 -1
- data/lib/2.7/fast_mmaped_file_rs.so +0 -0
- data/lib/3.0/fast_mmaped_file_rs.so +0 -0
- data/lib/3.1/fast_mmaped_file_rs.so +0 -0
- data/lib/3.2/fast_mmaped_file_rs.so +0 -0
- data/lib/prometheus/client/configuration.rb +2 -1
- data/lib/prometheus/client/formats/text.rb +2 -24
- data/lib/prometheus/client/helper/loader.rb +40 -0
- data/lib/prometheus/client/helper/mmaped_file.rb +12 -1
- data/lib/prometheus/client/rack/exporter.rb +3 -1
- data/lib/prometheus/client/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 96d81412cc07d51c07e2a4d3d63d24a0977f8f3218a469bace79a9d2abc929b6
|
4
|
+
data.tar.gz: 61c024688e436092d09d4b77214603e1c374be69e6b4b289d0d2997f10218c24
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a030d17f52d1530a30e8ff892632a00ff8cd10489ded1a7268eca58898dfd73c627f56f38bd8382066c70d9e58400719e37f1485790fcea723d377080512712c
|
7
|
+
data.tar.gz: 6a2bf596a09c5a7a255051f4c6945f2f325d6d0db03b0a9f537af50ade9b1b17356240628723401391b8eaa27ad66e2d9b7730c9c65b5a040049d19663b429b1
|
data/README.md
CHANGED
@@ -200,6 +200,17 @@ Set `prometheus_multiproc_dir` environment variable to the path where you want m
|
|
200
200
|
prometheus_multiproc_dir=/tmp
|
201
201
|
```
|
202
202
|
|
203
|
+
### Multiprocess metrics via Rust extension
|
204
|
+
|
205
|
+
If the environment variable `prometheus_rust_multiprocess_metrics=true` is set or if the `rust_multiprocess_metrics`
|
206
|
+
configuration setting is `true` and the `fast_mmaped_file_rs` extension is available, it will be used to generate
|
207
|
+
multiprocess metrics. This should be significantly faster than the C extension.
|
208
|
+
|
209
|
+
### Read and write metrics via Rust extension
|
210
|
+
|
211
|
+
If the environment variable `prometheus_rust_mmaped_file=true` is set then if the `fast_mmaped_file_rs`
|
212
|
+
extension is available it will be used to read and write metrics from the mmapped file.
|
213
|
+
|
203
214
|
## Pitfalls
|
204
215
|
|
205
216
|
### PID cardinality
|
@@ -9,7 +9,7 @@ use crate::util;
|
|
9
9
|
use crate::PROM_EPARSING_ERROR;
|
10
10
|
|
11
11
|
/// A lightweight representation of Ruby ExceptionClasses.
|
12
|
-
#[derive(PartialEq, Clone, Copy, Debug)]
|
12
|
+
#[derive(PartialEq, Eq, Clone, Copy, Debug)]
|
13
13
|
pub enum RubyError {
|
14
14
|
Arg,
|
15
15
|
Encoding,
|
@@ -45,7 +45,7 @@ impl From<RubyError> for magnus::ExceptionClass {
|
|
45
45
|
/// Errors returned internally within the crate. Methods called directly by Ruby return
|
46
46
|
/// `magnus::error::Error` as do functions that interact heavily with Ruby. This can be
|
47
47
|
/// converted into a `magnus::error::Error` at the boundary between Rust and Ruby.
|
48
|
-
#[derive(PartialEq, Error, Debug)]
|
48
|
+
#[derive(PartialEq, Eq, Error, Debug)]
|
49
49
|
pub enum MmapError {
|
50
50
|
/// A read or write was made while another thread had mutable access to the mmap.
|
51
51
|
#[error("read/write operation attempted while mmap was being written to")]
|
@@ -1,4 +1,4 @@
|
|
1
|
-
use magnus::
|
1
|
+
use magnus::Symbol;
|
2
2
|
use std::fmt::Write;
|
3
3
|
use std::str;
|
4
4
|
|
@@ -80,7 +80,7 @@ impl<'a> BorrowedData<'a> {
|
|
80
80
|
#[derive(Clone, Debug)]
|
81
81
|
pub struct EntryMetadata {
|
82
82
|
pub multiprocess_mode: Symbol,
|
83
|
-
pub type_:
|
83
|
+
pub type_: Symbol,
|
84
84
|
pub value: f64,
|
85
85
|
}
|
86
86
|
|
@@ -176,7 +176,7 @@ impl FileEntry {
|
|
176
176
|
out.push_str(family_name);
|
177
177
|
out.push(' ');
|
178
178
|
|
179
|
-
out.push_str(self.meta.type_.name().expect("name was invalid UTF-8"));
|
179
|
+
out.push_str(&self.meta.type_.name().expect("name was invalid UTF-8"));
|
180
180
|
out.push('\n');
|
181
181
|
}
|
182
182
|
|
@@ -438,7 +438,7 @@ mod test {
|
|
438
438
|
path,
|
439
439
|
len: case.json.len(),
|
440
440
|
multiprocess_mode: Symbol::new(case.multiprocess_mode),
|
441
|
-
type_:
|
441
|
+
type_: Symbol::new("gauge"),
|
442
442
|
pid: pid.to_string(),
|
443
443
|
};
|
444
444
|
file_infos.push(info);
|
@@ -544,7 +544,7 @@ mod test {
|
|
544
544
|
path,
|
545
545
|
len: json.len(),
|
546
546
|
multiprocess_mode: Symbol::new(case.multiprocess_mode),
|
547
|
-
type_:
|
547
|
+
type_: Symbol::new(case.metric_type),
|
548
548
|
pid: "worker-1".to_string(),
|
549
549
|
};
|
550
550
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
use magnus::exception::*;
|
2
|
-
use magnus::{Error, RString,
|
2
|
+
use magnus::{Error, RString, Symbol, Value};
|
3
3
|
use std::ffi::OsString;
|
4
4
|
use std::fs::File;
|
5
5
|
use std::io::{self, Read, Seek};
|
@@ -18,7 +18,7 @@ pub struct FileInfo {
|
|
18
18
|
pub path: PathBuf,
|
19
19
|
pub len: usize,
|
20
20
|
pub multiprocess_mode: Symbol,
|
21
|
-
pub type_:
|
21
|
+
pub type_: Symbol,
|
22
22
|
pub pid: String,
|
23
23
|
}
|
24
24
|
|
@@ -59,7 +59,7 @@ impl FileInfo {
|
|
59
59
|
let multiprocess_mode = Symbol::from_value(params[1])
|
60
60
|
.ok_or_else(|| err!(arg_error(), "expected multiprocess_mode to be a symbol"))?;
|
61
61
|
|
62
|
-
let type_ =
|
62
|
+
let type_ = Symbol::from_value(params[2])
|
63
63
|
.ok_or_else(|| err!(arg_error(), "expected file type to be a symbol"))?;
|
64
64
|
|
65
65
|
let pid = RString::from_value(params[3])
|
@@ -103,7 +103,7 @@ impl FileInfo {
|
|
103
103
|
|
104
104
|
#[cfg(test)]
|
105
105
|
mod test {
|
106
|
-
use magnus::{eval, RArray,
|
106
|
+
use magnus::{eval, RArray, Symbol};
|
107
107
|
use rand::{thread_rng, Rng};
|
108
108
|
use sha2::{Digest, Sha256};
|
109
109
|
|
@@ -168,7 +168,7 @@ mod test {
|
|
168
168
|
path,
|
169
169
|
len: buf.len(),
|
170
170
|
multiprocess_mode: Symbol::new("puma"),
|
171
|
-
type_:
|
171
|
+
type_: Symbol::new("max"),
|
172
172
|
pid: "worker-0_0".to_string(),
|
173
173
|
};
|
174
174
|
|
@@ -166,7 +166,7 @@ impl EntryMap {
|
|
166
166
|
|
167
167
|
#[cfg(test)]
|
168
168
|
mod test {
|
169
|
-
use magnus::
|
169
|
+
use magnus::Symbol;
|
170
170
|
use std::mem;
|
171
171
|
|
172
172
|
use super::*;
|
@@ -197,7 +197,7 @@ mod test {
|
|
197
197
|
},
|
198
198
|
meta: EntryMetadata {
|
199
199
|
multiprocess_mode: Symbol::new("max"),
|
200
|
-
type_:
|
200
|
+
type_: Symbol::new("gauge"),
|
201
201
|
value: 1.0,
|
202
202
|
},
|
203
203
|
},
|
@@ -208,7 +208,7 @@ mod test {
|
|
208
208
|
},
|
209
209
|
meta: EntryMetadata {
|
210
210
|
multiprocess_mode: Symbol::new("max"),
|
211
|
-
type_:
|
211
|
+
type_: Symbol::new("gauge"),
|
212
212
|
value: 1.0,
|
213
213
|
},
|
214
214
|
},
|
@@ -219,7 +219,7 @@ mod test {
|
|
219
219
|
},
|
220
220
|
meta: EntryMetadata {
|
221
221
|
multiprocess_mode: Symbol::new("max"),
|
222
|
-
type_:
|
222
|
+
type_: Symbol::new("gauge"),
|
223
223
|
value: 1.0,
|
224
224
|
},
|
225
225
|
},
|
@@ -230,7 +230,7 @@ mod test {
|
|
230
230
|
},
|
231
231
|
meta: EntryMetadata {
|
232
232
|
multiprocess_mode: Symbol::new("max"),
|
233
|
-
type_:
|
233
|
+
type_: Symbol::new("gauge"),
|
234
234
|
value: 1.0,
|
235
235
|
},
|
236
236
|
},
|
@@ -241,7 +241,7 @@ mod test {
|
|
241
241
|
},
|
242
242
|
meta: EntryMetadata {
|
243
243
|
multiprocess_mode: Symbol::new("all"),
|
244
|
-
type_:
|
244
|
+
type_: Symbol::new("gauge"),
|
245
245
|
value: 1.0,
|
246
246
|
},
|
247
247
|
},
|
@@ -252,7 +252,7 @@ mod test {
|
|
252
252
|
},
|
253
253
|
meta: EntryMetadata {
|
254
254
|
multiprocess_mode: Symbol::new("all"),
|
255
|
-
type_:
|
255
|
+
type_: Symbol::new("gauge"),
|
256
256
|
value: 1.0,
|
257
257
|
},
|
258
258
|
},
|
@@ -293,7 +293,7 @@ mod test {
|
|
293
293
|
},
|
294
294
|
meta: EntryMetadata {
|
295
295
|
multiprocess_mode: Symbol::new("all"),
|
296
|
-
type_:
|
296
|
+
type_: Symbol::new("gauge"),
|
297
297
|
value: 1.0,
|
298
298
|
},
|
299
299
|
};
|
@@ -305,7 +305,7 @@ mod test {
|
|
305
305
|
},
|
306
306
|
meta: EntryMetadata {
|
307
307
|
multiprocess_mode: Symbol::new("all"),
|
308
|
-
type_:
|
308
|
+
type_: Symbol::new("gauge"),
|
309
309
|
value: 5.0,
|
310
310
|
},
|
311
311
|
};
|
@@ -317,7 +317,7 @@ mod test {
|
|
317
317
|
},
|
318
318
|
meta: EntryMetadata {
|
319
319
|
multiprocess_mode: Symbol::new("all"),
|
320
|
-
type_:
|
320
|
+
type_: Symbol::new("gauge"),
|
321
321
|
value: 100.0,
|
322
322
|
},
|
323
323
|
};
|
@@ -329,7 +329,7 @@ mod test {
|
|
329
329
|
},
|
330
330
|
meta: EntryMetadata {
|
331
331
|
multiprocess_mode: Symbol::new("all"),
|
332
|
-
type_:
|
332
|
+
type_: Symbol::new("gauge"),
|
333
333
|
value: 1.0,
|
334
334
|
},
|
335
335
|
};
|
@@ -460,7 +460,7 @@ mod test {
|
|
460
460
|
path,
|
461
461
|
len: case.json.len(),
|
462
462
|
multiprocess_mode: Symbol::new("max"),
|
463
|
-
type_:
|
463
|
+
type_: Symbol::new("gauge"),
|
464
464
|
pid: "worker-1".to_string(),
|
465
465
|
};
|
466
466
|
|
@@ -204,6 +204,12 @@ impl InnerMmap {
|
|
204
204
|
self.map.as_ptr()
|
205
205
|
}
|
206
206
|
|
207
|
+
/// Returns a mutable raw pointer to the mmap.
|
208
|
+
/// For use in updating RString internals which requires a mutable pointer.
|
209
|
+
pub fn as_mut_ptr(&self) -> *mut u8 {
|
210
|
+
self.map.as_ptr().cast_mut()
|
211
|
+
}
|
212
|
+
|
207
213
|
/// Perform an msync(2) on the mmap, flushing all changes written
|
208
214
|
/// to disk. The sync may optionally be performed asynchronously.
|
209
215
|
pub fn flush(&mut self, f_async: bool) -> Result<()> {
|
@@ -3,8 +3,8 @@ use magnus::prelude::*;
|
|
3
3
|
use magnus::rb_sys::{AsRawValue, FromRawValue};
|
4
4
|
use magnus::typed_data::Obj;
|
5
5
|
use magnus::value::Fixnum;
|
6
|
-
use magnus::{
|
7
|
-
use nix::libc::c_long;
|
6
|
+
use magnus::{eval, scan_args, Error, Integer, RArray, RClass, RHash, RString, Value};
|
7
|
+
use nix::libc::{c_char, c_long, c_ulong};
|
8
8
|
use rb_sys::rb_str_new_static;
|
9
9
|
use std::fs::File;
|
10
10
|
use std::io::{prelude::*, SeekFrom};
|
@@ -25,6 +25,11 @@ use inner::InnerMmap;
|
|
25
25
|
|
26
26
|
mod inner;
|
27
27
|
|
28
|
+
/// The Ruby `STR_NOEMBED` flag, aka `FL_USER1`.
|
29
|
+
const STR_NOEMBED: c_ulong = 1 << (13);
|
30
|
+
/// The Ruby `STR_SHARED` flag, aka `FL_USER2`.
|
31
|
+
const STR_SHARED: c_ulong = 1 << (14);
|
32
|
+
|
28
33
|
/// A Rust struct wrapped in a Ruby object, providing access to a memory-mapped
|
29
34
|
/// file used to store, update, and read out Prometheus metrics.
|
30
35
|
///
|
@@ -151,13 +156,28 @@ impl MmapedFile {
|
|
151
156
|
///
|
152
157
|
/// return a substring of <em>lenght</em> characters from <em>start</em>
|
153
158
|
pub fn slice(rb_self: Obj<Self>, args: &[Value]) -> magnus::error::Result<RString> {
|
154
|
-
// The C
|
159
|
+
// The C implementation would trigger a GC cycle via `rb_gc_force_recycle`
|
155
160
|
// if the `MM_PROTECT` flag is set, but in practice this is never used.
|
156
161
|
// We omit this logic, particularly because `rb_gc_force_recycle` is a
|
157
162
|
// no-op as of Ruby 3.1.
|
163
|
+
let rs_self = &*rb_self;
|
164
|
+
|
165
|
+
let str = rs_self.str(rb_self)?;
|
166
|
+
rs_self._slice(rb_self, str, args)
|
167
|
+
}
|
158
168
|
|
159
|
-
|
160
|
-
|
169
|
+
fn _slice(
|
170
|
+
&self,
|
171
|
+
rb_self: Obj<Self>,
|
172
|
+
str: RString,
|
173
|
+
args: &[Value],
|
174
|
+
) -> magnus::error::Result<RString> {
|
175
|
+
let substr: RString = str.funcall("[]", args)?;
|
176
|
+
|
177
|
+
// Track shared child strings which use the same backing storage.
|
178
|
+
if Self::rb_string_is_shared(substr) {
|
179
|
+
(*rb_self).track_rstring(rb_self, substr)?;
|
180
|
+
}
|
161
181
|
|
162
182
|
// The C implementation does this, perhaps to validate that the len we
|
163
183
|
// provided is actually being used.
|
@@ -166,7 +186,7 @@ impl MmapedFile {
|
|
166
186
|
Ok(())
|
167
187
|
})?;
|
168
188
|
|
169
|
-
|
189
|
+
Ok(substr)
|
170
190
|
}
|
171
191
|
|
172
192
|
/// Document-method: msync
|
@@ -223,7 +243,8 @@ impl MmapedFile {
|
|
223
243
|
})?;
|
224
244
|
|
225
245
|
// Update each String object to be zero-length.
|
226
|
-
rs_self.
|
246
|
+
let cap = util::cast_chk::<_, c_long>(rs_self.capacity(), "capacity")?;
|
247
|
+
rs_self.update_weak_map(rb_self, rs_self.as_mut_ptr(), cap)?;
|
227
248
|
|
228
249
|
// Remove the `InnerMmap` from the `RwLock`. This will drop
|
229
250
|
// end of this function, unmapping and closing the file.
|
@@ -341,16 +362,19 @@ impl MmapedFile {
|
|
341
362
|
Ok(unsafe { rb_str_new_static(ptr as _, len as _) })
|
342
363
|
})?;
|
343
364
|
|
365
|
+
// SAFETY: We know that rb_str_new_static returns a VALUE.
|
344
366
|
let val = unsafe { Value::from_raw(val_id) };
|
345
367
|
|
346
368
|
// UNWRAP: We created this value as a string above.
|
347
369
|
let str = RString::from_value(val).unwrap();
|
348
370
|
|
349
|
-
|
371
|
+
// Freeze the root string so it can't be mutated out from under any
|
372
|
+
// substrings created. This object is never exposed to callers.
|
373
|
+
str.freeze();
|
350
374
|
|
351
|
-
//
|
352
|
-
|
353
|
-
|
375
|
+
// Track the RString in our `WeakMap` so we can update its address if
|
376
|
+
// we re-mmap the backing file.
|
377
|
+
(*rb_self).track_rstring(rb_self, str)?;
|
354
378
|
|
355
379
|
Ok(str)
|
356
380
|
}
|
@@ -358,23 +382,20 @@ impl MmapedFile {
|
|
358
382
|
/// If we reallocate, any live Ruby strings provided by the `str()` method
|
359
383
|
/// will be invalidated. We need to iterate over them using and update their
|
360
384
|
/// heap pointers to the newly allocated memory region.
|
361
|
-
fn update_weak_map(
|
385
|
+
fn update_weak_map(
|
386
|
+
&self,
|
387
|
+
rb_self: Obj<Self>,
|
388
|
+
old_ptr: *const c_char,
|
389
|
+
old_cap: c_long,
|
390
|
+
) -> magnus::error::Result<()> {
|
362
391
|
let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
|
363
392
|
|
364
|
-
let
|
365
|
-
// Pointers are not `Send`, convert it to usize to allow capture in closure.
|
366
|
-
let new_ptr = inner.as_ptr() as usize;
|
367
|
-
let new_len = util::cast_chk::<_, c_long>(inner.len(), "mmap len")?;
|
393
|
+
let new_len = self.inner(|inner| util::cast_chk::<_, c_long>(inner.len(), "mmap len"))?;
|
368
394
|
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
let block = Proc::from_fn(move |args, _block| {
|
374
|
-
let val = args
|
375
|
-
.get(0)
|
376
|
-
.ok_or_else(|| err!(arg_error(), "no argument received from `each_value`"))?;
|
377
|
-
let str = RString::from_value(*val)
|
395
|
+
// Iterate over the values of the `WeakMap`.
|
396
|
+
for val in tracker.enumeratorize("each_value", ()) {
|
397
|
+
let rb_string = val?;
|
398
|
+
let str = RString::from_value(rb_string)
|
378
399
|
.ok_or_else(|| err!(arg_error(), "weakmap value was not a string"))?;
|
379
400
|
|
380
401
|
// SAFETY: We're messing with Ruby's internals here, YOLO.
|
@@ -383,18 +404,42 @@ impl MmapedFile {
|
|
383
404
|
// which provides access to its internals.
|
384
405
|
let mut raw_str = Self::rb_string_internal(str);
|
385
406
|
|
407
|
+
// Shared string have their own `ptr` and `len` values, but `aux`
|
408
|
+
// is the id of the parent string so the GC can track this
|
409
|
+
// dependency. The `ptr` will always be an offset from the base
|
410
|
+
// address of the mmap, and `len` will be the length of the mmap
|
411
|
+
// less the offset from the base.
|
412
|
+
if Self::rb_string_is_shared(str) && new_len > 0 {
|
413
|
+
// Calculate how far into the original mmap the shared string
|
414
|
+
// started and update to the equivalent address in the new
|
415
|
+
// one.
|
416
|
+
let substr_ptr = raw_str.as_ref().as_.heap.ptr;
|
417
|
+
let offset = substr_ptr.offset_from(old_ptr);
|
418
|
+
|
419
|
+
raw_str.as_mut().as_.heap.ptr = self.as_mut_ptr().offset(offset);
|
420
|
+
|
421
|
+
let current_len = raw_str.as_ref().as_.heap.len;
|
422
|
+
let new_shared_len = old_cap + current_len;
|
423
|
+
|
424
|
+
raw_str.as_mut().as_.heap.len = new_shared_len;
|
425
|
+
continue;
|
426
|
+
}
|
427
|
+
|
386
428
|
// Update the string to point to the new mmapped file.
|
387
429
|
// We're matching the behavior of Ruby's `str_new_static` function.
|
388
430
|
// See https://github.com/ruby/ruby/blob/e51014f9c05aa65cbf203442d37fef7c12390015/string.c#L1030-L1053
|
389
|
-
|
431
|
+
//
|
432
|
+
// We deliberately do _NOT_ increment the `capa` field of the
|
433
|
+
// string to match the new `len`. We were initially doing this,
|
434
|
+
// but consistently triggered GCs in the middle of updating the
|
435
|
+
// string pointers, causing a segfault.
|
436
|
+
//
|
437
|
+
// See https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/issues/45
|
438
|
+
raw_str.as_mut().as_.heap.ptr = self.as_mut_ptr();
|
390
439
|
raw_str.as_mut().as_.heap.len = new_len;
|
391
|
-
raw_str.as_mut().as_.heap.aux.capa = new_len;
|
392
440
|
}
|
393
|
-
|
394
|
-
});
|
441
|
+
}
|
395
442
|
|
396
|
-
// Execute the block.
|
397
|
-
let _: Value = tracker.funcall_with_block("each_value", (), block)?;
|
398
443
|
Ok(())
|
399
444
|
}
|
400
445
|
|
@@ -429,6 +474,9 @@ impl MmapedFile {
|
|
429
474
|
}
|
430
475
|
|
431
476
|
if new_cap != self.capacity() {
|
477
|
+
let old_ptr = self.as_mut_ptr();
|
478
|
+
let old_cap = util::cast_chk::<_, c_long>(self.capacity(), "capacity")?;
|
479
|
+
|
432
480
|
// Drop the old mmap.
|
433
481
|
let (mut file, path) = self.take_inner()?.munmap();
|
434
482
|
|
@@ -439,7 +487,7 @@ impl MmapedFile {
|
|
439
487
|
|
440
488
|
self.insert_inner(new_inner)?;
|
441
489
|
|
442
|
-
return self.update_weak_map(rb_self);
|
490
|
+
return self.update_weak_map(rb_self, old_ptr, old_cap);
|
443
491
|
}
|
444
492
|
|
445
493
|
Ok(())
|
@@ -476,6 +524,15 @@ impl MmapedFile {
|
|
476
524
|
Ok(())
|
477
525
|
}
|
478
526
|
|
527
|
+
fn track_rstring(&self, rb_self: Obj<Self>, str: RString) -> magnus::error::Result<()> {
|
528
|
+
let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
|
529
|
+
|
530
|
+
// Use the string's Id as the key in the `WeakMap`.
|
531
|
+
let key = str.as_raw();
|
532
|
+
let _: Value = tracker.funcall("[]=", (key, str))?;
|
533
|
+
Ok(())
|
534
|
+
}
|
535
|
+
|
479
536
|
/// The total capacity of the underlying mmap.
|
480
537
|
#[inline]
|
481
538
|
fn capacity(&self) -> usize {
|
@@ -489,6 +546,13 @@ impl MmapedFile {
|
|
489
546
|
.map_err(|e| e.into())
|
490
547
|
}
|
491
548
|
|
549
|
+
fn as_mut_ptr(&self) -> *mut c_char {
|
550
|
+
// UNWRAP: This is actually infallible, but we need to
|
551
|
+
// wrap it in a `Result` for use with `inner()`.
|
552
|
+
self.inner(|inner| Ok(inner.as_mut_ptr() as *mut c_char))
|
553
|
+
.unwrap()
|
554
|
+
}
|
555
|
+
|
492
556
|
/// Takes a closure with immutable access to InnerMmap. Will fail if the inner
|
493
557
|
/// object has a mutable borrow or has been dropped.
|
494
558
|
fn inner<F, T>(&self, func: F) -> Result<T>
|
@@ -544,9 +608,27 @@ impl MmapedFile {
|
|
544
608
|
Ok(())
|
545
609
|
}
|
546
610
|
|
611
|
+
/// Check if an RString is shared. Shared string use the same underlying
|
612
|
+
/// storage as their parent, taking an offset from the start. By default
|
613
|
+
/// they must run to the end of the parent string.
|
614
|
+
fn rb_string_is_shared(rb_str: RString) -> bool {
|
615
|
+
// SAFETY: We only hold a reference to the raw object for the duration
|
616
|
+
// of this function, and no Ruby code is called.
|
617
|
+
let flags = unsafe {
|
618
|
+
let raw_str = Self::rb_string_internal(rb_str);
|
619
|
+
raw_str.as_ref().basic.flags
|
620
|
+
};
|
621
|
+
let shared_flags = STR_SHARED | STR_NOEMBED;
|
622
|
+
|
623
|
+
flags & shared_flags == shared_flags
|
624
|
+
}
|
625
|
+
|
547
626
|
/// Convert `magnus::RString` into the raw binding used by `rb_sys::RString`.
|
548
627
|
/// We need this to manually change the pointer and length values for strings
|
549
628
|
/// when moving the mmap to a new file.
|
629
|
+
///
|
630
|
+
/// SAFETY: Calling Ruby code while the returned object is held may result
|
631
|
+
/// in it being mutated or dropped.
|
550
632
|
unsafe fn rb_string_internal(rb_str: RString) -> NonNull<rb_sys::RString> {
|
551
633
|
mem::transmute::<RString, NonNull<rb_sys::RString>>(rb_str)
|
552
634
|
}
|
@@ -666,18 +748,101 @@ mod test {
|
|
666
748
|
let ruby = magnus::Ruby::get().unwrap();
|
667
749
|
crate::init(&ruby).unwrap();
|
668
750
|
|
751
|
+
fn assert_internals(
|
752
|
+
obj: Obj<MmapedFile>,
|
753
|
+
parent_id: c_ulong,
|
754
|
+
child_id: c_ulong,
|
755
|
+
unshared_id: c_ulong,
|
756
|
+
) {
|
757
|
+
let rs_self = &*obj;
|
758
|
+
let tracker: Value = obj.ivar_get("@weak_obj_tracker").unwrap();
|
759
|
+
|
760
|
+
let mmap_ptr = rs_self.as_mut_ptr();
|
761
|
+
let mmap_len = rs_self.capacity();
|
762
|
+
|
763
|
+
let mut parent_checked = false;
|
764
|
+
let mut child_checked = false;
|
765
|
+
|
766
|
+
for val in tracker.enumeratorize("each_value", ()) {
|
767
|
+
let rb_string = val.unwrap();
|
768
|
+
let str = RString::from_value(rb_string).unwrap();
|
769
|
+
|
770
|
+
unsafe {
|
771
|
+
let raw_str = MmapedFile::rb_string_internal(str);
|
772
|
+
if str.as_raw() == child_id {
|
773
|
+
assert_eq!(parent_id, raw_str.as_ref().as_.heap.aux.shared);
|
774
|
+
|
775
|
+
let child_offset =
|
776
|
+
mmap_len as isize - raw_str.as_ref().as_.heap.len as isize;
|
777
|
+
assert_eq!(mmap_ptr.offset(child_offset), raw_str.as_ref().as_.heap.ptr);
|
778
|
+
|
779
|
+
child_checked = true;
|
780
|
+
} else if str.as_raw() == parent_id {
|
781
|
+
assert_eq!(parent_id, str.as_raw());
|
782
|
+
|
783
|
+
assert_eq!(mmap_ptr, raw_str.as_ref().as_.heap.ptr);
|
784
|
+
assert_eq!(mmap_len as c_long, raw_str.as_ref().as_.heap.len);
|
785
|
+
assert!(raw_str.as_ref().basic.flags & (STR_SHARED | STR_NOEMBED) > 0);
|
786
|
+
assert!(str.is_frozen());
|
787
|
+
|
788
|
+
parent_checked = true;
|
789
|
+
} else if str.as_raw() == unshared_id {
|
790
|
+
panic!("tracking unshared string");
|
791
|
+
} else {
|
792
|
+
panic!("unknown string");
|
793
|
+
}
|
794
|
+
}
|
795
|
+
}
|
796
|
+
assert!(parent_checked && child_checked);
|
797
|
+
}
|
798
|
+
|
669
799
|
let obj = create_obj();
|
670
800
|
let _ = populate_entries(&obj);
|
671
801
|
|
672
802
|
let rs_self = &*obj;
|
673
803
|
|
674
|
-
|
675
|
-
let
|
676
|
-
|
677
|
-
|
804
|
+
// Create a string containing the full mmap.
|
805
|
+
let parent_str = rs_self.str(obj).unwrap();
|
806
|
+
let parent_id = parent_str.as_raw();
|
807
|
+
|
808
|
+
// Ruby's shared strings are only created when they go to the end of
|
809
|
+
// original string.
|
810
|
+
let len = rs_self.inner(|inner| Ok(inner.len())).unwrap();
|
811
|
+
let shareable_range = Range::new(1, len - 1, false).unwrap().as_value();
|
812
|
+
|
813
|
+
// This string should re-use the parent's buffer with an offset and have
|
814
|
+
// the parent's id in `as.heap.aux.shared`
|
815
|
+
let child_str = rs_self._slice(obj, parent_str, &[shareable_range]).unwrap();
|
816
|
+
let child_id = child_str.as_raw();
|
817
|
+
|
818
|
+
// A range that does not reach the end of the parent will not be shared.
|
819
|
+
assert!(len > 4);
|
820
|
+
let unshareable_range = Range::new(0, 4, false).unwrap().as_value();
|
821
|
+
|
822
|
+
// This string should NOT be tracked, it should own its own buffer.
|
823
|
+
let unshared_str = rs_self
|
824
|
+
._slice(obj, parent_str, &[unshareable_range])
|
825
|
+
.unwrap();
|
826
|
+
let unshared_id = unshared_str.as_raw();
|
827
|
+
assert!(!MmapedFile::rb_string_is_shared(unshared_str));
|
828
|
+
|
829
|
+
assert_internals(obj, parent_id, child_id, unshared_id);
|
830
|
+
|
831
|
+
let orig_ptr = rs_self.as_mut_ptr();
|
832
|
+
// Expand a bunch to ensure we remap
|
833
|
+
for _ in 0..16 {
|
834
|
+
rs_self.expand_to_fit(obj, rs_self.capacity() * 2).unwrap();
|
835
|
+
}
|
836
|
+
let new_ptr = rs_self.as_mut_ptr();
|
837
|
+
assert!(orig_ptr != new_ptr);
|
678
838
|
|
679
839
|
// If we haven't updated the pointer to the newly remapped file this will segfault.
|
680
|
-
let _: Value = eval!("puts
|
840
|
+
let _: Value = eval!("puts parent", parent = parent_str).unwrap();
|
841
|
+
let _: Value = eval!("puts child", child = child_str).unwrap();
|
842
|
+
let _: Value = eval!("puts unshared", unshared = unshared_str).unwrap();
|
843
|
+
|
844
|
+
// Confirm that tracked strings are still valid.
|
845
|
+
assert_internals(obj, parent_id, child_id, unshared_id);
|
681
846
|
}
|
682
847
|
|
683
848
|
#[test]
|
@@ -2,7 +2,7 @@ use smallvec::SmallVec;
|
|
2
2
|
use std::str;
|
3
3
|
|
4
4
|
/// String slices pointing to the fields of a borrowed `Entry`'s JSON data.
|
5
|
-
#[derive(PartialEq, Debug)]
|
5
|
+
#[derive(PartialEq, Eq, Debug)]
|
6
6
|
pub struct MetricText<'a> {
|
7
7
|
pub family_name: &'a str,
|
8
8
|
pub metric_name: &'a str,
|
@@ -10,14 +10,14 @@ pub struct MetricText<'a> {
|
|
10
10
|
pub values: SmallVec<[&'a str; 4]>,
|
11
11
|
}
|
12
12
|
|
13
|
-
#[derive(PartialEq, Debug)]
|
13
|
+
#[derive(PartialEq, Eq, Debug)]
|
14
14
|
struct MetricNames<'a> {
|
15
15
|
label_json: &'a str,
|
16
16
|
family_name: &'a str,
|
17
17
|
metric_name: &'a str,
|
18
18
|
}
|
19
19
|
|
20
|
-
#[derive(PartialEq, Debug)]
|
20
|
+
#[derive(PartialEq, Eq, Debug)]
|
21
21
|
struct MetricLabelVals<'a> {
|
22
22
|
labels: SmallVec<[&'a str; 4]>,
|
23
23
|
values: SmallVec<[&'a str; 4]>,
|
@@ -6,7 +6,7 @@ use crate::util::CheckedOps;
|
|
6
6
|
use crate::Result;
|
7
7
|
|
8
8
|
/// The logic to save a `MetricsEntry`, or parse one from a byte slice.
|
9
|
-
#[derive(PartialEq, Clone, Debug)]
|
9
|
+
#[derive(PartialEq, Eq, Clone, Debug)]
|
10
10
|
pub struct RawEntry<'a> {
|
11
11
|
bytes: &'a [u8],
|
12
12
|
encoded_len: usize,
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -7,13 +7,14 @@ require 'tmpdir'
|
|
7
7
|
module Prometheus
|
8
8
|
module Client
|
9
9
|
class Configuration
|
10
|
-
attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider
|
10
|
+
attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider, :rust_multiprocess_metrics
|
11
11
|
|
12
12
|
def initialize
|
13
13
|
@value_class = ::Prometheus::Client::MmapedValue
|
14
14
|
@initial_mmap_file_size = ::Prometheus::Client::PageSize.page_size(fallback_page_size: 4096)
|
15
15
|
@logger = Logger.new($stdout)
|
16
16
|
@pid_provider = Process.method(:pid)
|
17
|
+
@rust_multiprocess_metrics = ENV.fetch('prometheus_rust_multiprocess_metrics', nil) == 'true'
|
17
18
|
@multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do
|
18
19
|
Dir.mktmpdir("prometheus-mmap")
|
19
20
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'prometheus/client/uses_value_type'
|
2
2
|
require 'prometheus/client/helper/json_parser'
|
3
|
+
require 'prometheus/client/helper/loader'
|
3
4
|
require 'prometheus/client/helper/plain_file'
|
4
5
|
require 'prometheus/client/helper/metrics_processing'
|
5
6
|
require 'prometheus/client/helper/metrics_representation'
|
@@ -31,7 +32,7 @@ module Prometheus
|
|
31
32
|
.map {|f| Helper::PlainFile.new(f) }
|
32
33
|
.map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
|
33
34
|
|
34
|
-
if use_rust && rust_impl_available?
|
35
|
+
if use_rust && Prometheus::Client::Helper::Loader.rust_impl_available?
|
35
36
|
FastMmapedFileRs.to_metrics(file_list.to_a)
|
36
37
|
else
|
37
38
|
FastMmapedFile.to_metrics(file_list.to_a)
|
@@ -46,29 +47,6 @@ module Prometheus
|
|
46
47
|
|
47
48
|
private
|
48
49
|
|
49
|
-
def load_rust_extension
|
50
|
-
begin
|
51
|
-
ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
|
52
|
-
require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
|
53
|
-
rescue LoadError
|
54
|
-
require 'fast_mmaped_file_rs'
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def check_for_rust
|
59
|
-
# This will be evaluated on each invocation even with `||=` if
|
60
|
-
# `@rust_available` if false. Running a `require` statement is slow,
|
61
|
-
# so the `rust_impl_available?` method memoizes the result, external
|
62
|
-
# callers can only trigger this method a single time.
|
63
|
-
@rust_available = begin
|
64
|
-
load_rust_extension
|
65
|
-
true
|
66
|
-
rescue LoadError
|
67
|
-
Prometheus::Client.logger.info('FastMmapedFileRs unavailable')
|
68
|
-
false
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
50
|
def load_metrics(path)
|
73
51
|
metrics = {}
|
74
52
|
Dir.glob(File.join(path, '*.db')).sort.each do |f|
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Prometheus
|
2
|
+
module Client
|
3
|
+
module Helper
|
4
|
+
module Loader
|
5
|
+
class << self
|
6
|
+
def rust_impl_available?
|
7
|
+
return @rust_available unless @rust_available.nil?
|
8
|
+
|
9
|
+
check_for_rust
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def load_rust_extension
|
15
|
+
begin
|
16
|
+
ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
|
17
|
+
require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
|
18
|
+
rescue LoadError
|
19
|
+
require 'fast_mmaped_file_rs'
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def check_for_rust
|
24
|
+
# This will be evaluated on each invocation even with `||=` if
|
25
|
+
# `@rust_available` if false. Running a `require` statement is slow,
|
26
|
+
# so the `rust_impl_available?` method memoizes the result, external
|
27
|
+
# callers can only trigger this method a single time.
|
28
|
+
@rust_available = begin
|
29
|
+
load_rust_extension
|
30
|
+
true
|
31
|
+
rescue LoadError
|
32
|
+
Prometheus::Client.logger.info('FastMmapedFileRs unavailable')
|
33
|
+
false
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'prometheus/client/helper/entry_parser'
|
2
2
|
require 'prometheus/client/helper/file_locker'
|
3
|
+
require 'prometheus/client/helper/loader'
|
3
4
|
|
4
5
|
# load precompiled extension if available
|
5
6
|
begin
|
@@ -12,7 +13,17 @@ end
|
|
12
13
|
module Prometheus
|
13
14
|
module Client
|
14
15
|
module Helper
|
15
|
-
|
16
|
+
# We can't check `Prometheus::Client.configuration` as this creates a circular dependency
|
17
|
+
if (ENV.fetch('prometheus_rust_mmaped_file', nil) == "true" &&
|
18
|
+
Prometheus::Client::Helper::Loader.rust_impl_available?)
|
19
|
+
class MmapedFile < FastMmapedFileRs
|
20
|
+
end
|
21
|
+
else
|
22
|
+
class MmapedFile < FastMmapedFile
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
class MmapedFile
|
16
27
|
include EntryParser
|
17
28
|
|
18
29
|
attr_reader :filepath, :size
|
@@ -62,8 +62,10 @@ module Prometheus
|
|
62
62
|
end
|
63
63
|
|
64
64
|
def respond_with(format)
|
65
|
+
rust_enabled = Prometheus::Client.configuration.rust_multiprocess_metrics
|
66
|
+
|
65
67
|
response = if Prometheus::Client.configuration.value_class.multiprocess
|
66
|
-
format.marshal_multiprocess
|
68
|
+
format.marshal_multiprocess(use_rust: rust_enabled)
|
67
69
|
else
|
68
70
|
format.marshal
|
69
71
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: prometheus-client-mmap
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.24.3
|
5
5
|
platform: x86_64-linux
|
6
6
|
authors:
|
7
7
|
- Tobias Schmidt
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2023-05-
|
14
|
+
date: 2023-05-20 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: rb_sys
|
@@ -180,6 +180,7 @@ files:
|
|
180
180
|
- lib/prometheus/client/helper/entry_parser.rb
|
181
181
|
- lib/prometheus/client/helper/file_locker.rb
|
182
182
|
- lib/prometheus/client/helper/json_parser.rb
|
183
|
+
- lib/prometheus/client/helper/loader.rb
|
183
184
|
- lib/prometheus/client/helper/metrics_processing.rb
|
184
185
|
- lib/prometheus/client/helper/metrics_representation.rb
|
185
186
|
- lib/prometheus/client/helper/mmaped_file.rb
|