prometheus-client-mmap 0.19.1 → 0.20.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,579 @@
1
+ use magnus::{StaticSymbol, Symbol};
2
+ use std::fmt::Write;
3
+ use std::str;
4
+
5
+ use crate::error::{MmapError, RubyError};
6
+ use crate::file_info::FileInfo;
7
+ use crate::parser::{self, MetricText};
8
+ use crate::raw_entry::RawEntry;
9
+ use crate::Result;
10
+ use crate::{SYM_GAUGE, SYM_LIVESUM, SYM_MAX, SYM_MIN};
11
+
12
+ /// A metrics entry extracted from a `*.db` file.
13
+ #[derive(Clone, Debug)]
14
+ pub struct FileEntry {
15
+ pub data: EntryData,
16
+ pub meta: EntryMetadata,
17
+ }
18
+
19
+ /// The primary data payload for a `FileEntry`, the JSON string and the
20
+ /// associated pid, if significant. Used as the key for `EntryMap`.
21
+ #[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Debug)]
22
+ pub struct EntryData {
23
+ pub json: String,
24
+ pub pid: Option<String>,
25
+ }
26
+
27
+ impl<'a> PartialEq<BorrowedData<'a>> for EntryData {
28
+ fn eq(&self, other: &BorrowedData) -> bool {
29
+ self.pid.as_deref() == other.pid && self.json == other.json
30
+ }
31
+ }
32
+
33
+ impl<'a> TryFrom<BorrowedData<'a>> for EntryData {
34
+ type Error = MmapError;
35
+
36
+ fn try_from(borrowed: BorrowedData) -> Result<Self> {
37
+ let mut json = String::new();
38
+ if json.try_reserve_exact(borrowed.json.len()).is_err() {
39
+ return Err(MmapError::OutOfMemory(borrowed.json.len()));
40
+ }
41
+ json.push_str(borrowed.json);
42
+
43
+ Ok(Self {
44
+ json,
45
+ // Don't bother checking for allocation failure, typically ~10 bytes
46
+ pid: borrowed.pid.map(|p| p.to_string()),
47
+ })
48
+ }
49
+ }
50
+
51
+ /// A borrowed copy of the JSON string and pid for a `FileEntry`. We use this
52
+ /// to check if a given string/pid combination is present in the `EntryMap`,
53
+ /// copying them to owned values only when needed.
54
+ #[derive(Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Debug)]
55
+ pub struct BorrowedData<'a> {
56
+ pub json: &'a str,
57
+ pub pid: Option<&'a str>,
58
+ }
59
+
60
+ impl<'a> BorrowedData<'a> {
61
+ pub fn new(
62
+ raw_entry: &'a RawEntry,
63
+ file_info: &'a FileInfo,
64
+ pid_significant: bool,
65
+ ) -> Result<Self> {
66
+ let json = str::from_utf8(raw_entry.json())
67
+ .map_err(|e| MmapError::Encoding(format!("invalid UTF-8 in entry JSON: {e}")))?;
68
+
69
+ let pid = if pid_significant {
70
+ Some(file_info.pid.as_str())
71
+ } else {
72
+ None
73
+ };
74
+
75
+ Ok(Self { json, pid })
76
+ }
77
+ }
78
+
79
+ /// The metadata associated with a `FileEntry`. The value in `EntryMap`.
80
+ #[derive(Clone, Debug)]
81
+ pub struct EntryMetadata {
82
+ pub multiprocess_mode: Symbol,
83
+ pub type_: StaticSymbol,
84
+ pub value: f64,
85
+ }
86
+
87
+ impl EntryMetadata {
88
+ /// Construct a new `FileEntry`, copying the JSON string from the `RawEntry`
89
+ /// into an internal buffer.
90
+ pub fn new(mmap_entry: &RawEntry, file: &FileInfo) -> Result<Self> {
91
+ let value = mmap_entry.value();
92
+
93
+ Ok(EntryMetadata {
94
+ multiprocess_mode: file.multiprocess_mode,
95
+ type_: file.type_,
96
+ value,
97
+ })
98
+ }
99
+
100
+ /// Combine values with another `EntryMetadata`.
101
+ pub fn merge(&mut self, other: &Self) {
102
+ if self.type_ == SYM_GAUGE {
103
+ match self.multiprocess_mode {
104
+ s if s == SYM_MIN => self.value = self.value.min(other.value),
105
+ s if s == SYM_MAX => self.value = self.value.max(other.value),
106
+ s if s == SYM_LIVESUM => self.value += other.value,
107
+ _ => self.value = other.value,
108
+ }
109
+ } else {
110
+ self.value += other.value;
111
+ }
112
+ }
113
+
114
+ /// Validate if pid is significant for metric.
115
+ pub fn is_pid_significant(&self) -> bool {
116
+ let mp = self.multiprocess_mode;
117
+
118
+ self.type_ == SYM_GAUGE && !(mp == SYM_MIN || mp == SYM_MAX || mp == SYM_LIVESUM)
119
+ }
120
+ }
121
+
122
+ impl FileEntry {
123
+ /// Convert the sorted entries into a String in Prometheus metrics format.
124
+ pub fn entries_to_string(entries: Vec<FileEntry>) -> Result<String> {
125
+ // We guesstimate that lines are ~100 bytes long, preallocate the string to
126
+ // roughly that size.
127
+ let mut out = String::new();
128
+ out.try_reserve(entries.len() * 128)
129
+ .map_err(|_| MmapError::OutOfMemory(entries.len() * 128))?;
130
+
131
+ let mut prev_name: Option<String> = None;
132
+
133
+ let entry_count = entries.len();
134
+ let mut processed_count = 0;
135
+
136
+ for entry in entries {
137
+ let metrics_data = match parser::parse_metrics(&entry.data.json) {
138
+ Some(m) => m,
139
+ // We don't exit the function here so the total number of invalid
140
+ // entries can be calculated below.
141
+ None => continue,
142
+ };
143
+
144
+ match prev_name.as_ref() {
145
+ Some(p) if p == metrics_data.family_name => {}
146
+ _ => {
147
+ entry.append_header(metrics_data.family_name, &mut out);
148
+ prev_name = Some(metrics_data.family_name.to_owned());
149
+ }
150
+ }
151
+
152
+ entry.append_entry(metrics_data, &mut out)?;
153
+
154
+ writeln!(&mut out, " {}", entry.meta.value)
155
+ .map_err(|e| MmapError::Other(format!("Failed to append to output: {e}")))?;
156
+
157
+ processed_count += 1;
158
+ }
159
+
160
+ if processed_count != entry_count {
161
+ return Err(MmapError::legacy(
162
+ format!("Processed entries {processed_count} != map entries {entry_count}"),
163
+ RubyError::Runtime,
164
+ ));
165
+ }
166
+
167
+ Ok(out)
168
+ }
169
+
170
+ fn append_header(&self, family_name: &str, out: &mut String) {
171
+ out.push_str("# HELP ");
172
+ out.push_str(family_name);
173
+ out.push_str(" Multiprocess metric\n");
174
+
175
+ out.push_str("# TYPE ");
176
+ out.push_str(family_name);
177
+ out.push(' ');
178
+
179
+ out.push_str(self.meta.type_.name().expect("name was invalid UTF-8"));
180
+ out.push('\n');
181
+ }
182
+
183
+ fn append_entry(&self, json_data: MetricText, out: &mut String) -> Result<()> {
184
+ out.push_str(json_data.metric_name);
185
+
186
+ if json_data.labels.is_empty() {
187
+ if let Some(pid) = self.data.pid.as_ref() {
188
+ out.push_str("{pid=\"");
189
+ out.push_str(pid);
190
+ out.push_str("\"}");
191
+ }
192
+
193
+ return Ok(());
194
+ }
195
+
196
+ out.push('{');
197
+
198
+ let it = json_data.labels.iter().zip(json_data.values.iter());
199
+
200
+ for (i, (&key, &val)) in it.enumerate() {
201
+ out.push_str(key);
202
+
203
+ out.push_str("=\"");
204
+
205
+ // `null` values will be output as `""`.
206
+ if val != "null" {
207
+ out.push_str(val);
208
+ }
209
+ out.push('"');
210
+
211
+ if i < json_data.labels.len() - 1 {
212
+ out.push(',');
213
+ }
214
+ }
215
+
216
+ if let Some(pid) = self.data.pid.as_ref() {
217
+ out.push_str(",pid=\"");
218
+ out.push_str(pid);
219
+ out.push('"');
220
+ }
221
+
222
+ out.push('}');
223
+
224
+ Ok(())
225
+ }
226
+ }
227
+
228
+ #[cfg(test)]
229
+ mod test {
230
+ use bstr::BString;
231
+ use indoc::indoc;
232
+
233
+ use super::*;
234
+ use crate::file_info::FileInfo;
235
+ use crate::raw_entry::RawEntry;
236
+ use crate::testhelper::{TestEntry, TestFile};
237
+
238
+ #[test]
239
+ fn test_entries_to_string() {
240
+ struct TestCase {
241
+ name: &'static str,
242
+ multiprocess_mode: &'static str,
243
+ json: &'static [&'static str],
244
+ values: &'static [f64],
245
+ pids: &'static [&'static str],
246
+ expected_out: Option<&'static str>,
247
+ expected_err: Option<MmapError>,
248
+ }
249
+
250
+ let _cleanup = unsafe { magnus::embed::init() };
251
+ let ruby = magnus::Ruby::get().unwrap();
252
+ crate::init(&ruby).unwrap();
253
+
254
+ let tc = vec![
255
+ TestCase {
256
+ name: "one metric, pid significant",
257
+ multiprocess_mode: "all",
258
+ json: &[r#"["family","name",["label_a","label_b"],["value_a","value_b"]]"#],
259
+ values: &[1.0],
260
+ pids: &["worker-1"],
261
+ expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
262
+ # TYPE family gauge
263
+ name{label_a="value_a",label_b="value_b",pid="worker-1"} 1
264
+ "##}),
265
+ expected_err: None,
266
+ },
267
+ TestCase {
268
+ name: "one metric, no pid",
269
+ multiprocess_mode: "min",
270
+ json: &[r#"["family","name",["label_a","label_b"],["value_a","value_b"]]"#],
271
+ values: &[1.0],
272
+ pids: &["worker-1"],
273
+ expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
274
+ # TYPE family gauge
275
+ name{label_a="value_a",label_b="value_b"} 1
276
+ "##}),
277
+ expected_err: None,
278
+ },
279
+ TestCase {
280
+ name: "floating point shown",
281
+ multiprocess_mode: "min",
282
+ json: &[r#"["family","name",["label_a","label_b"],["value_a","value_b"]]"#],
283
+ values: &[1.5],
284
+ pids: &["worker-1"],
285
+ expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
286
+ # TYPE family gauge
287
+ name{label_a="value_a",label_b="value_b"} 1.5
288
+ "##}),
289
+ expected_err: None,
290
+ },
291
+ TestCase {
292
+ name: "no labels, pid significant",
293
+ multiprocess_mode: "all",
294
+ json: &[r#"["family","name",[],[]]"#],
295
+ values: &[1.0],
296
+ pids: &["worker-1"],
297
+ expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
298
+ # TYPE family gauge
299
+ name{pid="worker-1"} 1
300
+ "##}),
301
+ expected_err: None,
302
+ },
303
+ TestCase {
304
+ name: "no labels, no pid",
305
+ multiprocess_mode: "min",
306
+ json: &[r#"["family","name",[],[]]"#],
307
+ values: &[1.0],
308
+ pids: &["worker-1"],
309
+ expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
310
+ # TYPE family gauge
311
+ name 1
312
+ "##}),
313
+ expected_err: None,
314
+ },
315
+ TestCase {
316
+ name: "two metrics, same family, pid significant",
317
+ multiprocess_mode: "all",
318
+ json: &[
319
+ r#"["family","first",["label_a","label_b"],["value_a","value_b"]]"#,
320
+ r#"["family","second",["label_a","label_b"],["value_a","value_b"]]"#,
321
+ ],
322
+ values: &[1.0, 2.0],
323
+ pids: &["worker-1", "worker-1"],
324
+ expected_out: Some(indoc! {r##"# HELP family Multiprocess metric
325
+ # TYPE family gauge
326
+ first{label_a="value_a",label_b="value_b",pid="worker-1"} 1
327
+ second{label_a="value_a",label_b="value_b",pid="worker-1"} 2
328
+ "##}),
329
+ expected_err: None,
330
+ },
331
+ TestCase {
332
+ name: "two metrics, different family, pid significant",
333
+ multiprocess_mode: "min",
334
+ json: &[
335
+ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#,
336
+ r#"["second_family","second_name",["label_a","label_b"],["value_a","value_b"]]"#,
337
+ ],
338
+ values: &[1.0, 2.0],
339
+ pids: &["worker-1", "worker-1"],
340
+ expected_out: Some(indoc! {r##"# HELP first_family Multiprocess metric
341
+ # TYPE first_family gauge
342
+ first_name{label_a="value_a",label_b="value_b"} 1
343
+ # HELP second_family Multiprocess metric
344
+ # TYPE second_family gauge
345
+ second_name{label_a="value_a",label_b="value_b"} 2
346
+ "##}),
347
+ expected_err: None,
348
+ },
349
+ TestCase {
350
+ name: "three metrics, two different families, pid significant",
351
+ multiprocess_mode: "all",
352
+ json: &[
353
+ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#,
354
+ r#"["first_family","second_name",["label_a","label_b"],["value_a","value_b"]]"#,
355
+ r#"["second_family","second_name",["label_a","label_b"],["value_a","value_b"]]"#,
356
+ ],
357
+ values: &[1.0, 2.0, 3.0],
358
+ pids: &["worker-1", "worker-1", "worker-1"],
359
+ expected_out: Some(indoc! {r##"# HELP first_family Multiprocess metric
360
+ # TYPE first_family gauge
361
+ first_name{label_a="value_a",label_b="value_b",pid="worker-1"} 1
362
+ second_name{label_a="value_a",label_b="value_b",pid="worker-1"} 2
363
+ # HELP second_family Multiprocess metric
364
+ # TYPE second_family gauge
365
+ second_name{label_a="value_a",label_b="value_b",pid="worker-1"} 3
366
+ "##}),
367
+ expected_err: None,
368
+ },
369
+ TestCase {
370
+ name: "same metrics, pid significant, separate workers",
371
+ multiprocess_mode: "all",
372
+ json: &[
373
+ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#,
374
+ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#,
375
+ ],
376
+ values: &[1.0, 2.0],
377
+ pids: &["worker-1", "worker-2"],
378
+ expected_out: Some(indoc! {r##"# HELP first_family Multiprocess metric
379
+ # TYPE first_family gauge
380
+ first_name{label_a="value_a",label_b="value_b",pid="worker-1"} 1
381
+ first_name{label_a="value_a",label_b="value_b",pid="worker-2"} 2
382
+ "##}),
383
+ expected_err: None,
384
+ },
385
+ TestCase {
386
+ name: "same metrics, pid not significant, separate workers",
387
+ multiprocess_mode: "max",
388
+ json: &[
389
+ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#,
390
+ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#,
391
+ ],
392
+ values: &[1.0, 2.0],
393
+ pids: &["worker-1", "worker-2"],
394
+ expected_out: Some(indoc! {r##"# HELP first_family Multiprocess metric
395
+ # TYPE first_family gauge
396
+ first_name{label_a="value_a",label_b="value_b"} 1
397
+ first_name{label_a="value_a",label_b="value_b"} 2
398
+ "##}),
399
+ expected_err: None,
400
+ },
401
+ TestCase {
402
+ name: "entry fails to parse",
403
+ multiprocess_mode: "min",
404
+ json: &[
405
+ r#"["first_family","first_name",["label_a","label_b"],["value_a","value_b"]]"#,
406
+ r#"[not valid"#,
407
+ ],
408
+ values: &[1.0, 2.0],
409
+ pids: &["worker-1", "worker-1"],
410
+ expected_out: None,
411
+ expected_err: Some(MmapError::legacy(
412
+ "Processed entries 1 != map entries 2".to_owned(),
413
+ RubyError::Runtime,
414
+ )),
415
+ },
416
+ ];
417
+
418
+ for case in tc {
419
+ let name = case.name;
420
+
421
+ let input_bytes: Vec<BString> = case
422
+ .json
423
+ .iter()
424
+ .zip(case.values)
425
+ .map(|(&s, &value)| TestEntry::new(s, value).as_bstring())
426
+ .collect();
427
+
428
+ let mut file_infos = Vec::new();
429
+ for pid in case.pids {
430
+ let TestFile {
431
+ file,
432
+ path,
433
+ dir: _dir,
434
+ } = TestFile::new(b"foobar");
435
+
436
+ let info = FileInfo {
437
+ file,
438
+ path,
439
+ len: case.json.len(),
440
+ multiprocess_mode: Symbol::new(case.multiprocess_mode),
441
+ type_: StaticSymbol::new("gauge"),
442
+ pid: pid.to_string(),
443
+ };
444
+ file_infos.push(info);
445
+ }
446
+
447
+ let file_entries: Vec<FileEntry> = input_bytes
448
+ .iter()
449
+ .map(|s| RawEntry::from_slice(s).unwrap())
450
+ .zip(file_infos)
451
+ .map(|(entry, info)| {
452
+ let meta = EntryMetadata::new(&entry, &info).unwrap();
453
+ let borrowed =
454
+ BorrowedData::new(&entry, &info, meta.is_pid_significant()).unwrap();
455
+ let data = EntryData::try_from(borrowed).unwrap();
456
+ FileEntry { data, meta }
457
+ })
458
+ .collect();
459
+
460
+ let output = FileEntry::entries_to_string(file_entries);
461
+
462
+ if let Some(expected_out) = case.expected_out {
463
+ assert_eq!(
464
+ expected_out,
465
+ output.as_ref().unwrap(),
466
+ "test case: {name} - output"
467
+ );
468
+ }
469
+
470
+ if let Some(expected_err) = case.expected_err {
471
+ assert_eq!(
472
+ expected_err,
473
+ output.unwrap_err(),
474
+ "test case: {name} - error"
475
+ );
476
+ }
477
+ }
478
+ }
479
+
480
+ #[test]
481
+ fn test_merge() {
482
+ struct TestCase {
483
+ name: &'static str,
484
+ metric_type: &'static str,
485
+ multiprocess_mode: &'static str,
486
+ values: &'static [f64],
487
+ expected_value: f64,
488
+ }
489
+
490
+ let _cleanup = unsafe { magnus::embed::init() };
491
+ let ruby = magnus::Ruby::get().unwrap();
492
+ crate::init(&ruby).unwrap();
493
+
494
+ let tc = vec![
495
+ TestCase {
496
+ name: "gauge max",
497
+ metric_type: "gauge",
498
+ multiprocess_mode: "max",
499
+ values: &[1.0, 5.0],
500
+ expected_value: 5.0,
501
+ },
502
+ TestCase {
503
+ name: "gauge min",
504
+ metric_type: "gauge",
505
+ multiprocess_mode: "min",
506
+ values: &[1.0, 5.0],
507
+ expected_value: 1.0,
508
+ },
509
+ TestCase {
510
+ name: "gauge livesum",
511
+ metric_type: "gauge",
512
+ multiprocess_mode: "livesum",
513
+ values: &[1.0, 5.0],
514
+ expected_value: 6.0,
515
+ },
516
+ TestCase {
517
+ name: "gauge all",
518
+ metric_type: "gauge",
519
+ multiprocess_mode: "all",
520
+ values: &[1.0, 5.0],
521
+ expected_value: 5.0,
522
+ },
523
+ TestCase {
524
+ name: "not a gauge",
525
+ metric_type: "histogram",
526
+ multiprocess_mode: "max",
527
+ values: &[1.0, 5.0],
528
+ expected_value: 6.0,
529
+ },
530
+ ];
531
+
532
+ for case in tc {
533
+ let name = case.name;
534
+ let json = r#"["family","metric",["label_a","label_b"],["value_a","value_b"]]"#;
535
+
536
+ let TestFile {
537
+ file,
538
+ path,
539
+ dir: _dir,
540
+ } = TestFile::new(b"foobar");
541
+
542
+ let info = FileInfo {
543
+ file,
544
+ path,
545
+ len: json.len(),
546
+ multiprocess_mode: Symbol::new(case.multiprocess_mode),
547
+ type_: StaticSymbol::new(case.metric_type),
548
+ pid: "worker-1".to_string(),
549
+ };
550
+
551
+ let input_bytes: Vec<BString> = case
552
+ .values
553
+ .iter()
554
+ .map(|&value| TestEntry::new(json, value).as_bstring())
555
+ .collect();
556
+
557
+ let entries: Vec<FileEntry> = input_bytes
558
+ .iter()
559
+ .map(|s| RawEntry::from_slice(s).unwrap())
560
+ .map(|entry| {
561
+ let meta = EntryMetadata::new(&entry, &info).unwrap();
562
+ let borrowed =
563
+ BorrowedData::new(&entry, &info, meta.is_pid_significant()).unwrap();
564
+ let data = EntryData::try_from(borrowed).unwrap();
565
+ FileEntry { data, meta }
566
+ })
567
+ .collect();
568
+
569
+ let mut entry_a = entries[0].clone();
570
+ let entry_b = entries[1].clone();
571
+ entry_a.meta.merge(&entry_b.meta);
572
+
573
+ assert_eq!(
574
+ case.expected_value, entry_a.meta.value,
575
+ "test case: {name} - value"
576
+ );
577
+ }
578
+ }
579
+ }