prometheus-client-mmap 0.7.0.beta39 → 0.7.0.beta40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,76 @@
1
+ #ifndef __JSMN_H_
2
+ #define __JSMN_H_
3
+
4
+ #include <stddef.h>
5
+
6
+ #ifdef __cplusplus
7
+ extern "C" {
8
+ #endif
9
+
10
+ /**
11
+ * JSON type identifier. Basic types are:
12
+ * o Object
13
+ * o Array
14
+ * o String
15
+ * o Other primitive: number, boolean (true/false) or null
16
+ */
17
+ typedef enum {
18
+ JSMN_UNDEFINED = 0,
19
+ JSMN_OBJECT = 1,
20
+ JSMN_ARRAY = 2,
21
+ JSMN_STRING = 3,
22
+ JSMN_PRIMITIVE = 4
23
+ } jsmntype_t;
24
+
25
+ enum jsmnerr {
26
+ /* Not enough tokens were provided */
27
+ JSMN_ERROR_NOMEM = -1,
28
+ /* Invalid character inside JSON string */
29
+ JSMN_ERROR_INVAL = -2,
30
+ /* The string is not a full JSON packet, more bytes expected */
31
+ JSMN_ERROR_PART = -3
32
+ };
33
+
34
+ /**
35
+ * JSON token description.
36
+ * type type (object, array, string etc.)
37
+ * start start position in JSON data string
38
+ * end end position in JSON data string
39
+ */
40
+ typedef struct {
41
+ jsmntype_t type;
42
+ int start;
43
+ int end;
44
+ int size;
45
+ #ifdef JSMN_PARENT_LINKS
46
+ int parent;
47
+ #endif
48
+ } jsmntok_t;
49
+
50
+ /**
51
+ * JSON parser. Contains an array of token blocks available. Also stores
52
+ * the string being parsed now and current position in that string
53
+ */
54
+ typedef struct {
55
+ unsigned int pos; /* offset in the JSON string */
56
+ unsigned int toknext; /* next token to allocate */
57
+ int toksuper; /* superior token node, e.g parent object or array */
58
+ } jsmn_parser;
59
+
60
+ /**
61
+ * Create JSON parser over an array of tokens
62
+ */
63
+ void jsmn_init(jsmn_parser *parser);
64
+
65
+ /**
66
+ * Run JSON parser. It parses a JSON data string into and array of tokens, each describing
67
+ * a single JSON object.
68
+ */
69
+ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
70
+ jsmntok_t *tokens, unsigned int num_tokens);
71
+
72
+ #ifdef __cplusplus
73
+ }
74
+ #endif
75
+
76
+ #endif /* __JSMN_H_ */
@@ -1,6 +1,8 @@
1
1
  #ifndef MMAP_H
2
2
  #define MMAP_H
3
3
 
4
+ #include <ruby.h>
5
+
4
6
  #define MM_MODIFY 1
5
7
  #define MM_ORIGIN 2
6
8
  #define MM_CHANGE (MM_MODIFY | 4)
@@ -0,0 +1,92 @@
1
+ #include "rendering.h"
2
+ #include <jsmn.h>
3
+ #include <file_parsing.h>
4
+ #include <utils.h>
5
+
6
+ int not_null(const entry_struct *entry, const jsmntok_t *t){
7
+ return strncmp("null", entry->json + t->start, min(4, t->end - t->start)) != 0;
8
+ }
9
+
10
+ inline void append_token(VALUE string, const entry_struct *entry, const jsmntok_t *t){
11
+ rb_str_cat(string, entry->json + t->start, t->end - t->start); //TODO: add token sanity checks
12
+ }
13
+
14
+ void append_entry(VALUE string, const entry_struct *entry){
15
+ jsmn_parser parser;
16
+ jsmn_init(&parser);
17
+
18
+ jsmntok_t t[128]; //TODO: document how many labels we can process
19
+ int r = jsmn_parse(&parser, entry->json, entry->json_size, t, sizeof(t)/sizeof(t[0]));
20
+
21
+ if (r < 3) {
22
+ return; //TODO: better handle this case
23
+ }
24
+
25
+ //TODO: check (r - 4) % 2 == 0
26
+ int label_cnt = (r-5)/2;
27
+
28
+ append_token(string, entry, &t[2]);
29
+ if (label_cnt > 0){
30
+ rb_str_cat(string, "{", 1);
31
+
32
+ for(int i = 0; i < label_cnt; i++){
33
+ int key = 4 + i;
34
+ int val = 5 + label_cnt + i;
35
+ //TODO: add type checks
36
+ append_token(string, entry, &t[key]);
37
+ rb_str_cat(string, "=", 1);
38
+
39
+ rb_str_cat(string, "\"", 1);
40
+ if (not_null(entry, &t[val])){
41
+ append_token(string, entry, &t[val]);
42
+ }
43
+ rb_str_cat(string, "\"", 1);
44
+
45
+ if (i < label_cnt - 1){
46
+ rb_str_cat(string, ",", 1);
47
+ }
48
+ }
49
+ rb_str_cat(string, "}", 1);
50
+ }
51
+
52
+ char value[255];
53
+
54
+ //TODO: fix value representation %g doesn't math what Go expects
55
+ int written = snprintf(value, sizeof(value), " %g\n", entry->value);
56
+ rb_str_cat(string, value, written);
57
+ }
58
+
59
+ void append_entry_head(VALUE string, const entry_struct *entry){
60
+ static const char help_beg[] = "# HELP ";
61
+ static const char help_fin[] = " Multiprocess metric\n";
62
+
63
+ rb_str_cat(string, help_beg, sizeof(help_beg) - 1);
64
+ rb_str_cat(string, entry->name, entry->name_len);
65
+ rb_str_cat(string, help_fin, sizeof(help_fin) - 1);
66
+
67
+ static const char type_beg[] = "# TYPE ";
68
+
69
+ rb_str_cat(string, type_beg, sizeof(type_beg) - 1);
70
+ rb_str_cat(string, entry->name, entry->name_len);
71
+ rb_str_cat(string, " ", 1);
72
+ rb_str_cat2(string, rb_id2name(entry->type));
73
+ rb_str_cat(string, "\n", 1);
74
+ }
75
+
76
+ VALUE entries_to_string(entry_struct **sorted_entries, size_t entries_count){
77
+ VALUE rv = rb_str_new("", 0);
78
+
79
+ entry_struct *previous = NULL;
80
+
81
+ for(size_t i = 0; i < entries_count; i ++){
82
+ entry_struct *entry = sorted_entries[i];
83
+ if (!previous || previous->name_len != entry->name_len || strncmp(entry->name, previous->name, entry->name_len) != 0) {
84
+ previous = entry;
85
+ append_entry_head(rv, entry);
86
+ }
87
+
88
+ append_entry(rv, entry);
89
+ }
90
+
91
+ return rv;
92
+ }
@@ -0,0 +1,8 @@
1
+ #ifndef RENDERING_H
2
+ #define RENDERING_H
3
+ #include <ruby.h>
4
+ #include <file_parsing.h>
5
+
6
+ VALUE entries_to_string(entry_struct **sorted_entries, size_t entries_count);
7
+
8
+ #endif
@@ -0,0 +1,25 @@
1
+ #include <ruby.h>
2
+ #include "utils.h"
3
+
4
+ void save_exception(VALUE exception, const char *fmt, ...){
5
+ va_list args;
6
+
7
+ va_start(args, fmt);
8
+ VALUE message = rb_vsprintf(fmt, args);
9
+ va_end(args);
10
+
11
+ VALUE current_thread = rb_thread_current();
12
+ rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception"), exception);
13
+ rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception_message"), message);
14
+ }
15
+
16
+ void raise_last_exception(){
17
+ VALUE current_thread = rb_thread_current();
18
+ VALUE exception = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception"));
19
+ VALUE message = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception_message"));
20
+ if (exception != Qnil) {
21
+ rb_raise(exception, "%s", StringValueCStr(message));
22
+ } else {
23
+ rb_raise(rb_eRuntimeError, "no exception found in thread local");
24
+ }
25
+ }
@@ -0,0 +1,26 @@
1
+ #ifndef UNUSED_H
2
+ #define UNUSED_H
3
+
4
+ #ifdef UNUSED
5
+ #elif defined(__GNUC__)
6
+ #define UNUSED(x) UNUSED_ ## x __attribute__((unused))
7
+ #elif defined(__LCLINT__)
8
+ #define UNUSED(x) /*@unused@*/ x
9
+ #else
10
+ #define UNUSED(x) x
11
+ #endif
12
+
13
+ #define min(a,b) \
14
+ ({ __typeof__ (a) _a = (a); \
15
+ __typeof__ (b) _b = (b); \
16
+ _a < _b ? _a : _b; })
17
+
18
+ #define max(a,b) \
19
+ ({ __typeof__ (a) _a = (a); \
20
+ __typeof__ (b) _b = (b); \
21
+ _a > _b ? _a : _b; })
22
+
23
+ void raise_last_exception();
24
+ void save_exception(VALUE exception, const char *fmt, ...);
25
+
26
+ #endif
@@ -0,0 +1,160 @@
1
+ #include <sys/mman.h>
2
+ #include <unistd.h>
3
+ #include <fcntl.h>
4
+ #include <errno.h>
5
+
6
+ #include "file_format.h"
7
+ #include "mmap.h"
8
+ #include "value_access.h"
9
+
10
+ int open_and_extend_file(mm_ipc *i_mm, size_t len) {
11
+ int fd;
12
+
13
+ if ((fd = open(i_mm->t->path, i_mm->t->smode)) == -1) {
14
+ rb_raise(rb_eArgError, "Can't open %s", i_mm->t->path);
15
+ }
16
+
17
+ if (lseek(fd, len - i_mm->t->len - 1, SEEK_END) == -1) {
18
+ close(fd);
19
+ rb_raise(rb_eIOError, "Can't lseek %lu", len - i_mm->t->len - 1);
20
+ }
21
+
22
+ if (write(fd, "\000", 1) != 1) {
23
+ close(fd);
24
+ rb_raise(rb_eIOError, "Can't extend %s", i_mm->t->path);
25
+ }
26
+
27
+ return fd;
28
+ }
29
+
30
+ void expand(mm_ipc *i_mm, size_t len) {
31
+ if (len < i_mm->t->len) {
32
+ rb_raise(rb_eArgError, "Can't reduce the size of mmap");
33
+ }
34
+
35
+ if (munmap(i_mm->t->addr, i_mm->t->len)) {
36
+ rb_raise(rb_eArgError, "munmap failed");
37
+ }
38
+
39
+ int fd = open_and_extend_file(i_mm, len);
40
+
41
+ i_mm->t->addr = mmap(0, len, i_mm->t->pmode, i_mm->t->vscope, fd, i_mm->t->offset);
42
+
43
+ if (i_mm->t->addr == MAP_FAILED) {
44
+ close(fd);
45
+ rb_raise(rb_eArgError, "mmap failed");
46
+ }
47
+
48
+ if (close(fd) == -1) {
49
+ rb_raise(rb_eArgError, "Can't close %s", i_mm->t->path);
50
+ }
51
+
52
+ if ((i_mm->t->flag & MM_LOCK) && mlock(i_mm->t->addr, len) == -1) {
53
+ rb_raise(rb_eArgError, "mlock(%d)", errno);
54
+ }
55
+ i_mm->t->len = len;
56
+ i_mm->t->real = len;
57
+ }
58
+
59
+ void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
60
+ uint32_t key_length = (uint32_t)RSTRING_LEN(key);
61
+
62
+ char *pos = (char *)i_mm->t->addr + offset;
63
+
64
+ memcpy(pos, &key_length, sizeof(uint32_t));
65
+ pos += sizeof(uint32_t);
66
+
67
+ memmove(pos, StringValuePtr(key), key_length);
68
+ pos += key_length;
69
+
70
+ memset(pos, ' ', padding_length(key_length));
71
+ pos += padding_length(key_length);
72
+
73
+ double val = NUM2DBL(value);
74
+ memcpy(pos, &val, sizeof(double));
75
+ }
76
+
77
+ inline uint32_t load_used(mm_ipc *i_mm) {
78
+ uint32_t used = *((uint32_t *)i_mm->t->addr);
79
+
80
+ if (used == 0) {
81
+ used = START_POSITION;
82
+ }
83
+ return used;
84
+ }
85
+
86
+ inline void save_used(mm_ipc *i_mm, uint32_t used) {
87
+ *((uint32_t *)i_mm->t->addr) = used;
88
+ }
89
+
90
+ VALUE method_load_used(VALUE self) {
91
+ mm_ipc *i_mm;
92
+
93
+ GET_MMAP(self, i_mm, MM_MODIFY);
94
+ return UINT2NUM(load_used(i_mm));
95
+ }
96
+
97
+ VALUE method_save_used(VALUE self, VALUE value) {
98
+ Check_Type(value, T_FIXNUM);
99
+ mm_ipc *i_mm;
100
+
101
+ GET_MMAP(self, i_mm, MM_MODIFY);
102
+
103
+ if (i_mm->t->len < INITIAL_SIZE) {
104
+ expand(i_mm, INITIAL_SIZE);
105
+ }
106
+
107
+ save_used(i_mm, NUM2UINT(value));
108
+ return value;
109
+ }
110
+
111
+ VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
112
+ Check_Type(positions, T_HASH);
113
+ Check_Type(key, T_STRING);
114
+
115
+ VALUE position = rb_hash_lookup(positions, key);
116
+ if (position != Qnil){
117
+ return position;
118
+ }
119
+
120
+ mm_ipc *i_mm;
121
+ GET_MMAP(self, i_mm, MM_MODIFY);
122
+
123
+ if (i_mm->t->flag & MM_FROZEN) {
124
+ rb_error_frozen("mmap");
125
+ }
126
+
127
+ if (RSTRING_LEN(key) > UINT32_MAX) {
128
+ rb_raise(rb_eArgError, "string length gt %u", UINT32_MAX);
129
+ }
130
+
131
+ uint32_t key_length = (uint32_t)RSTRING_LEN(key);
132
+ uint32_t value_offset = sizeof(uint32_t) + key_length + padding_length(key_length);
133
+ uint32_t entry_length = value_offset + sizeof(double);
134
+
135
+ uint32_t used = load_used(i_mm);
136
+ while (i_mm->t->len < (used + entry_length)) {
137
+ expand(i_mm, i_mm->t->len * 2);
138
+ }
139
+
140
+ save_entry(i_mm, used, key, value);
141
+ save_used(i_mm, used + entry_length);
142
+ return rb_hash_aset(positions, key, INT2NUM(used + value_offset));
143
+ }
144
+
145
+ VALUE method_get_double(VALUE self, VALUE index) {
146
+ mm_ipc *i_mm;
147
+ GET_MMAP(self, i_mm, MM_MODIFY);
148
+
149
+ Check_Type(index, T_FIXNUM);
150
+ size_t idx = NUM2UINT(index);
151
+
152
+ if ((i_mm->t->real + sizeof(double)) <= idx) {
153
+ rb_raise(rb_eIndexError, "index %ld out of string", idx);
154
+ }
155
+
156
+ double tmp;
157
+
158
+ memcpy(&tmp, (char *)i_mm->t->addr + idx, sizeof(double));
159
+ return DBL2NUM(tmp);
160
+ }
@@ -0,0 +1,14 @@
1
+ #ifndef VALUE_ACCESS_H
2
+ #define VALUE_ACCESS_H
3
+
4
+ #include <ruby.h>
5
+
6
+ VALUE method_load_used(VALUE self);
7
+
8
+ VALUE method_save_used(VALUE self, VALUE value);
9
+
10
+ VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value);
11
+
12
+ VALUE method_get_double(VALUE self, VALUE index);
13
+
14
+ #endif
Binary file
@@ -27,9 +27,11 @@ module Prometheus
27
27
  end
28
28
 
29
29
  def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir)
30
- metrics = load_metrics(path)
30
+ file_list = Dir.glob(File.join(path, '*.db')).sort
31
+ .map {|f| Helper::PlainFile.new(f) }
32
+ .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
31
33
 
32
- Helper::MetricsRepresentation.to_text(Helper::MetricsProcessing.merge_metrics(metrics))
34
+ FastMmapedFile.to_metrics(file_list.to_a)
33
35
  end
34
36
 
35
37
  private
@@ -8,17 +8,20 @@ module Prometheus
8
8
  include EntryParser
9
9
  attr_reader :filepath
10
10
 
11
+ def data
12
+ @data ||= File.read(filepath, mode: 'rb')
13
+ end
14
+
11
15
  def initialize(filepath)
12
16
  @filepath = filepath
13
- @data = File.read(filepath, mode: 'rb')
14
17
  end
15
18
 
16
19
  def slice(*args)
17
- @data.slice(*args)
20
+ data.slice(*args)
18
21
  end
19
22
 
20
23
  def size
21
- @data.length
24
+ data.length
22
25
  end
23
26
  end
24
27
  end