prometheus-client-mmap 0.7.0.beta39 → 0.7.0.beta40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/fast_mmaped_file/extconf.rb +1 -1
- data/ext/fast_mmaped_file/fast_mmaped_file.c +74 -148
- data/ext/fast_mmaped_file/file_aggregation.c +0 -0
- data/ext/fast_mmaped_file/file_format.h +13 -0
- data/ext/fast_mmaped_file/file_parsing.c +222 -0
- data/ext/fast_mmaped_file/file_parsing.h +23 -0
- data/ext/fast_mmaped_file/file_reading.c +93 -0
- data/ext/fast_mmaped_file/file_reading.h +30 -0
- data/ext/fast_mmaped_file/globals.h +13 -0
- data/ext/fast_mmaped_file/hashmap.c +692 -0
- data/ext/fast_mmaped_file/hashmap.h +266 -0
- data/ext/fast_mmaped_file/jsmn.c +313 -0
- data/ext/fast_mmaped_file/jsmn.h +76 -0
- data/ext/fast_mmaped_file/mmap.h +2 -0
- data/ext/fast_mmaped_file/rendering.c +92 -0
- data/ext/fast_mmaped_file/rendering.h +8 -0
- data/ext/fast_mmaped_file/utils.c +25 -0
- data/ext/fast_mmaped_file/utils.h +26 -0
- data/ext/fast_mmaped_file/value_access.c +160 -0
- data/ext/fast_mmaped_file/value_access.h +14 -0
- data/lib/fast_mmaped_file.bundle +0 -0
- data/lib/prometheus/client/formats/text.rb +4 -2
- data/lib/prometheus/client/helper/plain_file.rb +6 -3
- data/lib/prometheus/client/version.rb +1 -1
- metadata +19 -2
@@ -0,0 +1,76 @@
|
|
1
|
+
#ifndef __JSMN_H_
|
2
|
+
#define __JSMN_H_
|
3
|
+
|
4
|
+
#include <stddef.h>
|
5
|
+
|
6
|
+
#ifdef __cplusplus
|
7
|
+
extern "C" {
|
8
|
+
#endif
|
9
|
+
|
10
|
+
/**
|
11
|
+
* JSON type identifier. Basic types are:
|
12
|
+
* o Object
|
13
|
+
* o Array
|
14
|
+
* o String
|
15
|
+
* o Other primitive: number, boolean (true/false) or null
|
16
|
+
*/
|
17
|
+
typedef enum {
|
18
|
+
JSMN_UNDEFINED = 0,
|
19
|
+
JSMN_OBJECT = 1,
|
20
|
+
JSMN_ARRAY = 2,
|
21
|
+
JSMN_STRING = 3,
|
22
|
+
JSMN_PRIMITIVE = 4
|
23
|
+
} jsmntype_t;
|
24
|
+
|
25
|
+
enum jsmnerr {
|
26
|
+
/* Not enough tokens were provided */
|
27
|
+
JSMN_ERROR_NOMEM = -1,
|
28
|
+
/* Invalid character inside JSON string */
|
29
|
+
JSMN_ERROR_INVAL = -2,
|
30
|
+
/* The string is not a full JSON packet, more bytes expected */
|
31
|
+
JSMN_ERROR_PART = -3
|
32
|
+
};
|
33
|
+
|
34
|
+
/**
|
35
|
+
* JSON token description.
|
36
|
+
* type type (object, array, string etc.)
|
37
|
+
* start start position in JSON data string
|
38
|
+
* end end position in JSON data string
|
39
|
+
*/
|
40
|
+
typedef struct {
|
41
|
+
jsmntype_t type;
|
42
|
+
int start;
|
43
|
+
int end;
|
44
|
+
int size;
|
45
|
+
#ifdef JSMN_PARENT_LINKS
|
46
|
+
int parent;
|
47
|
+
#endif
|
48
|
+
} jsmntok_t;
|
49
|
+
|
50
|
+
/**
|
51
|
+
* JSON parser. Contains an array of token blocks available. Also stores
|
52
|
+
* the string being parsed now and current position in that string
|
53
|
+
*/
|
54
|
+
typedef struct {
|
55
|
+
unsigned int pos; /* offset in the JSON string */
|
56
|
+
unsigned int toknext; /* next token to allocate */
|
57
|
+
int toksuper; /* superior token node, e.g parent object or array */
|
58
|
+
} jsmn_parser;
|
59
|
+
|
60
|
+
/**
|
61
|
+
* Create JSON parser over an array of tokens
|
62
|
+
*/
|
63
|
+
void jsmn_init(jsmn_parser *parser);
|
64
|
+
|
65
|
+
/**
|
66
|
+
* Run JSON parser. It parses a JSON data string into and array of tokens, each describing
|
67
|
+
* a single JSON object.
|
68
|
+
*/
|
69
|
+
int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
|
70
|
+
jsmntok_t *tokens, unsigned int num_tokens);
|
71
|
+
|
72
|
+
#ifdef __cplusplus
|
73
|
+
}
|
74
|
+
#endif
|
75
|
+
|
76
|
+
#endif /* __JSMN_H_ */
|
data/ext/fast_mmaped_file/mmap.h
CHANGED
@@ -0,0 +1,92 @@
|
|
1
|
+
#include "rendering.h"
|
2
|
+
#include <jsmn.h>
|
3
|
+
#include <file_parsing.h>
|
4
|
+
#include <utils.h>
|
5
|
+
|
6
|
+
int not_null(const entry_struct *entry, const jsmntok_t *t){
|
7
|
+
return strncmp("null", entry->json + t->start, min(4, t->end - t->start)) != 0;
|
8
|
+
}
|
9
|
+
|
10
|
+
inline void append_token(VALUE string, const entry_struct *entry, const jsmntok_t *t){
|
11
|
+
rb_str_cat(string, entry->json + t->start, t->end - t->start); //TODO: add token sanity checks
|
12
|
+
}
|
13
|
+
|
14
|
+
void append_entry(VALUE string, const entry_struct *entry){
|
15
|
+
jsmn_parser parser;
|
16
|
+
jsmn_init(&parser);
|
17
|
+
|
18
|
+
jsmntok_t t[128]; //TODO: document how many labels we can process
|
19
|
+
int r = jsmn_parse(&parser, entry->json, entry->json_size, t, sizeof(t)/sizeof(t[0]));
|
20
|
+
|
21
|
+
if (r < 3) {
|
22
|
+
return; //TODO: better handle this case
|
23
|
+
}
|
24
|
+
|
25
|
+
//TODO: check (r - 4) % 2 == 0
|
26
|
+
int label_cnt = (r-5)/2;
|
27
|
+
|
28
|
+
append_token(string, entry, &t[2]);
|
29
|
+
if (label_cnt > 0){
|
30
|
+
rb_str_cat(string, "{", 1);
|
31
|
+
|
32
|
+
for(int i = 0; i < label_cnt; i++){
|
33
|
+
int key = 4 + i;
|
34
|
+
int val = 5 + label_cnt + i;
|
35
|
+
//TODO: add type checks
|
36
|
+
append_token(string, entry, &t[key]);
|
37
|
+
rb_str_cat(string, "=", 1);
|
38
|
+
|
39
|
+
rb_str_cat(string, "\"", 1);
|
40
|
+
if (not_null(entry, &t[val])){
|
41
|
+
append_token(string, entry, &t[val]);
|
42
|
+
}
|
43
|
+
rb_str_cat(string, "\"", 1);
|
44
|
+
|
45
|
+
if (i < label_cnt - 1){
|
46
|
+
rb_str_cat(string, ",", 1);
|
47
|
+
}
|
48
|
+
}
|
49
|
+
rb_str_cat(string, "}", 1);
|
50
|
+
}
|
51
|
+
|
52
|
+
char value[255];
|
53
|
+
|
54
|
+
//TODO: fix value representation %g doesn't math what Go expects
|
55
|
+
int written = snprintf(value, sizeof(value), " %g\n", entry->value);
|
56
|
+
rb_str_cat(string, value, written);
|
57
|
+
}
|
58
|
+
|
59
|
+
void append_entry_head(VALUE string, const entry_struct *entry){
|
60
|
+
static const char help_beg[] = "# HELP ";
|
61
|
+
static const char help_fin[] = " Multiprocess metric\n";
|
62
|
+
|
63
|
+
rb_str_cat(string, help_beg, sizeof(help_beg) - 1);
|
64
|
+
rb_str_cat(string, entry->name, entry->name_len);
|
65
|
+
rb_str_cat(string, help_fin, sizeof(help_fin) - 1);
|
66
|
+
|
67
|
+
static const char type_beg[] = "# TYPE ";
|
68
|
+
|
69
|
+
rb_str_cat(string, type_beg, sizeof(type_beg) - 1);
|
70
|
+
rb_str_cat(string, entry->name, entry->name_len);
|
71
|
+
rb_str_cat(string, " ", 1);
|
72
|
+
rb_str_cat2(string, rb_id2name(entry->type));
|
73
|
+
rb_str_cat(string, "\n", 1);
|
74
|
+
}
|
75
|
+
|
76
|
+
VALUE entries_to_string(entry_struct **sorted_entries, size_t entries_count){
|
77
|
+
VALUE rv = rb_str_new("", 0);
|
78
|
+
|
79
|
+
entry_struct *previous = NULL;
|
80
|
+
|
81
|
+
for(size_t i = 0; i < entries_count; i ++){
|
82
|
+
entry_struct *entry = sorted_entries[i];
|
83
|
+
if (!previous || previous->name_len != entry->name_len || strncmp(entry->name, previous->name, entry->name_len) != 0) {
|
84
|
+
previous = entry;
|
85
|
+
append_entry_head(rv, entry);
|
86
|
+
}
|
87
|
+
|
88
|
+
append_entry(rv, entry);
|
89
|
+
}
|
90
|
+
|
91
|
+
return rv;
|
92
|
+
}
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#include <ruby.h>
|
2
|
+
#include "utils.h"
|
3
|
+
|
4
|
+
void save_exception(VALUE exception, const char *fmt, ...){
|
5
|
+
va_list args;
|
6
|
+
|
7
|
+
va_start(args, fmt);
|
8
|
+
VALUE message = rb_vsprintf(fmt, args);
|
9
|
+
va_end(args);
|
10
|
+
|
11
|
+
VALUE current_thread = rb_thread_current();
|
12
|
+
rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception"), exception);
|
13
|
+
rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception_message"), message);
|
14
|
+
}
|
15
|
+
|
16
|
+
void raise_last_exception(){
|
17
|
+
VALUE current_thread = rb_thread_current();
|
18
|
+
VALUE exception = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception"));
|
19
|
+
VALUE message = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception_message"));
|
20
|
+
if (exception != Qnil) {
|
21
|
+
rb_raise(exception, "%s", StringValueCStr(message));
|
22
|
+
} else {
|
23
|
+
rb_raise(rb_eRuntimeError, "no exception found in thread local");
|
24
|
+
}
|
25
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#ifndef UNUSED_H
|
2
|
+
#define UNUSED_H
|
3
|
+
|
4
|
+
#ifdef UNUSED
|
5
|
+
#elif defined(__GNUC__)
|
6
|
+
#define UNUSED(x) UNUSED_ ## x __attribute__((unused))
|
7
|
+
#elif defined(__LCLINT__)
|
8
|
+
#define UNUSED(x) /*@unused@*/ x
|
9
|
+
#else
|
10
|
+
#define UNUSED(x) x
|
11
|
+
#endif
|
12
|
+
|
13
|
+
#define min(a,b) \
|
14
|
+
({ __typeof__ (a) _a = (a); \
|
15
|
+
__typeof__ (b) _b = (b); \
|
16
|
+
_a < _b ? _a : _b; })
|
17
|
+
|
18
|
+
#define max(a,b) \
|
19
|
+
({ __typeof__ (a) _a = (a); \
|
20
|
+
__typeof__ (b) _b = (b); \
|
21
|
+
_a > _b ? _a : _b; })
|
22
|
+
|
23
|
+
void raise_last_exception();
|
24
|
+
void save_exception(VALUE exception, const char *fmt, ...);
|
25
|
+
|
26
|
+
#endif
|
@@ -0,0 +1,160 @@
|
|
1
|
+
#include <sys/mman.h>
|
2
|
+
#include <unistd.h>
|
3
|
+
#include <fcntl.h>
|
4
|
+
#include <errno.h>
|
5
|
+
|
6
|
+
#include "file_format.h"
|
7
|
+
#include "mmap.h"
|
8
|
+
#include "value_access.h"
|
9
|
+
|
10
|
+
int open_and_extend_file(mm_ipc *i_mm, size_t len) {
|
11
|
+
int fd;
|
12
|
+
|
13
|
+
if ((fd = open(i_mm->t->path, i_mm->t->smode)) == -1) {
|
14
|
+
rb_raise(rb_eArgError, "Can't open %s", i_mm->t->path);
|
15
|
+
}
|
16
|
+
|
17
|
+
if (lseek(fd, len - i_mm->t->len - 1, SEEK_END) == -1) {
|
18
|
+
close(fd);
|
19
|
+
rb_raise(rb_eIOError, "Can't lseek %lu", len - i_mm->t->len - 1);
|
20
|
+
}
|
21
|
+
|
22
|
+
if (write(fd, "\000", 1) != 1) {
|
23
|
+
close(fd);
|
24
|
+
rb_raise(rb_eIOError, "Can't extend %s", i_mm->t->path);
|
25
|
+
}
|
26
|
+
|
27
|
+
return fd;
|
28
|
+
}
|
29
|
+
|
30
|
+
void expand(mm_ipc *i_mm, size_t len) {
|
31
|
+
if (len < i_mm->t->len) {
|
32
|
+
rb_raise(rb_eArgError, "Can't reduce the size of mmap");
|
33
|
+
}
|
34
|
+
|
35
|
+
if (munmap(i_mm->t->addr, i_mm->t->len)) {
|
36
|
+
rb_raise(rb_eArgError, "munmap failed");
|
37
|
+
}
|
38
|
+
|
39
|
+
int fd = open_and_extend_file(i_mm, len);
|
40
|
+
|
41
|
+
i_mm->t->addr = mmap(0, len, i_mm->t->pmode, i_mm->t->vscope, fd, i_mm->t->offset);
|
42
|
+
|
43
|
+
if (i_mm->t->addr == MAP_FAILED) {
|
44
|
+
close(fd);
|
45
|
+
rb_raise(rb_eArgError, "mmap failed");
|
46
|
+
}
|
47
|
+
|
48
|
+
if (close(fd) == -1) {
|
49
|
+
rb_raise(rb_eArgError, "Can't close %s", i_mm->t->path);
|
50
|
+
}
|
51
|
+
|
52
|
+
if ((i_mm->t->flag & MM_LOCK) && mlock(i_mm->t->addr, len) == -1) {
|
53
|
+
rb_raise(rb_eArgError, "mlock(%d)", errno);
|
54
|
+
}
|
55
|
+
i_mm->t->len = len;
|
56
|
+
i_mm->t->real = len;
|
57
|
+
}
|
58
|
+
|
59
|
+
void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
|
60
|
+
uint32_t key_length = (uint32_t)RSTRING_LEN(key);
|
61
|
+
|
62
|
+
char *pos = (char *)i_mm->t->addr + offset;
|
63
|
+
|
64
|
+
memcpy(pos, &key_length, sizeof(uint32_t));
|
65
|
+
pos += sizeof(uint32_t);
|
66
|
+
|
67
|
+
memmove(pos, StringValuePtr(key), key_length);
|
68
|
+
pos += key_length;
|
69
|
+
|
70
|
+
memset(pos, ' ', padding_length(key_length));
|
71
|
+
pos += padding_length(key_length);
|
72
|
+
|
73
|
+
double val = NUM2DBL(value);
|
74
|
+
memcpy(pos, &val, sizeof(double));
|
75
|
+
}
|
76
|
+
|
77
|
+
inline uint32_t load_used(mm_ipc *i_mm) {
|
78
|
+
uint32_t used = *((uint32_t *)i_mm->t->addr);
|
79
|
+
|
80
|
+
if (used == 0) {
|
81
|
+
used = START_POSITION;
|
82
|
+
}
|
83
|
+
return used;
|
84
|
+
}
|
85
|
+
|
86
|
+
inline void save_used(mm_ipc *i_mm, uint32_t used) {
|
87
|
+
*((uint32_t *)i_mm->t->addr) = used;
|
88
|
+
}
|
89
|
+
|
90
|
+
VALUE method_load_used(VALUE self) {
|
91
|
+
mm_ipc *i_mm;
|
92
|
+
|
93
|
+
GET_MMAP(self, i_mm, MM_MODIFY);
|
94
|
+
return UINT2NUM(load_used(i_mm));
|
95
|
+
}
|
96
|
+
|
97
|
+
VALUE method_save_used(VALUE self, VALUE value) {
|
98
|
+
Check_Type(value, T_FIXNUM);
|
99
|
+
mm_ipc *i_mm;
|
100
|
+
|
101
|
+
GET_MMAP(self, i_mm, MM_MODIFY);
|
102
|
+
|
103
|
+
if (i_mm->t->len < INITIAL_SIZE) {
|
104
|
+
expand(i_mm, INITIAL_SIZE);
|
105
|
+
}
|
106
|
+
|
107
|
+
save_used(i_mm, NUM2UINT(value));
|
108
|
+
return value;
|
109
|
+
}
|
110
|
+
|
111
|
+
VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
|
112
|
+
Check_Type(positions, T_HASH);
|
113
|
+
Check_Type(key, T_STRING);
|
114
|
+
|
115
|
+
VALUE position = rb_hash_lookup(positions, key);
|
116
|
+
if (position != Qnil){
|
117
|
+
return position;
|
118
|
+
}
|
119
|
+
|
120
|
+
mm_ipc *i_mm;
|
121
|
+
GET_MMAP(self, i_mm, MM_MODIFY);
|
122
|
+
|
123
|
+
if (i_mm->t->flag & MM_FROZEN) {
|
124
|
+
rb_error_frozen("mmap");
|
125
|
+
}
|
126
|
+
|
127
|
+
if (RSTRING_LEN(key) > UINT32_MAX) {
|
128
|
+
rb_raise(rb_eArgError, "string length gt %u", UINT32_MAX);
|
129
|
+
}
|
130
|
+
|
131
|
+
uint32_t key_length = (uint32_t)RSTRING_LEN(key);
|
132
|
+
uint32_t value_offset = sizeof(uint32_t) + key_length + padding_length(key_length);
|
133
|
+
uint32_t entry_length = value_offset + sizeof(double);
|
134
|
+
|
135
|
+
uint32_t used = load_used(i_mm);
|
136
|
+
while (i_mm->t->len < (used + entry_length)) {
|
137
|
+
expand(i_mm, i_mm->t->len * 2);
|
138
|
+
}
|
139
|
+
|
140
|
+
save_entry(i_mm, used, key, value);
|
141
|
+
save_used(i_mm, used + entry_length);
|
142
|
+
return rb_hash_aset(positions, key, INT2NUM(used + value_offset));
|
143
|
+
}
|
144
|
+
|
145
|
+
VALUE method_get_double(VALUE self, VALUE index) {
|
146
|
+
mm_ipc *i_mm;
|
147
|
+
GET_MMAP(self, i_mm, MM_MODIFY);
|
148
|
+
|
149
|
+
Check_Type(index, T_FIXNUM);
|
150
|
+
size_t idx = NUM2UINT(index);
|
151
|
+
|
152
|
+
if ((i_mm->t->real + sizeof(double)) <= idx) {
|
153
|
+
rb_raise(rb_eIndexError, "index %ld out of string", idx);
|
154
|
+
}
|
155
|
+
|
156
|
+
double tmp;
|
157
|
+
|
158
|
+
memcpy(&tmp, (char *)i_mm->t->addr + idx, sizeof(double));
|
159
|
+
return DBL2NUM(tmp);
|
160
|
+
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
#ifndef VALUE_ACCESS_H
|
2
|
+
#define VALUE_ACCESS_H
|
3
|
+
|
4
|
+
#include <ruby.h>
|
5
|
+
|
6
|
+
VALUE method_load_used(VALUE self);
|
7
|
+
|
8
|
+
VALUE method_save_used(VALUE self, VALUE value);
|
9
|
+
|
10
|
+
VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value);
|
11
|
+
|
12
|
+
VALUE method_get_double(VALUE self, VALUE index);
|
13
|
+
|
14
|
+
#endif
|
data/lib/fast_mmaped_file.bundle
CHANGED
Binary file
|
@@ -27,9 +27,11 @@ module Prometheus
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir)
|
30
|
-
|
30
|
+
file_list = Dir.glob(File.join(path, '*.db')).sort
|
31
|
+
.map {|f| Helper::PlainFile.new(f) }
|
32
|
+
.map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
|
31
33
|
|
32
|
-
|
34
|
+
FastMmapedFile.to_metrics(file_list.to_a)
|
33
35
|
end
|
34
36
|
|
35
37
|
private
|
@@ -8,17 +8,20 @@ module Prometheus
|
|
8
8
|
include EntryParser
|
9
9
|
attr_reader :filepath
|
10
10
|
|
11
|
+
def data
|
12
|
+
@data ||= File.read(filepath, mode: 'rb')
|
13
|
+
end
|
14
|
+
|
11
15
|
def initialize(filepath)
|
12
16
|
@filepath = filepath
|
13
|
-
@data = File.read(filepath, mode: 'rb')
|
14
17
|
end
|
15
18
|
|
16
19
|
def slice(*args)
|
17
|
-
|
20
|
+
data.slice(*args)
|
18
21
|
end
|
19
22
|
|
20
23
|
def size
|
21
|
-
|
24
|
+
data.length
|
22
25
|
end
|
23
26
|
end
|
24
27
|
end
|