prometheus-client-mmap 0.7.0.beta41 → 0.7.0.beta42
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/fast_mmaped_file/extconf.rb +15 -0
- data/ext/fast_mmaped_file/fast_mmaped_file.c +49 -24
- data/ext/fast_mmaped_file/file_parsing.c +84 -109
- data/ext/fast_mmaped_file/file_parsing.h +7 -6
- data/ext/fast_mmaped_file/file_reading.c +41 -32
- data/ext/fast_mmaped_file/file_reading.h +1 -1
- data/ext/fast_mmaped_file/globals.h +3 -0
- data/ext/fast_mmaped_file/hashmap.c +409 -466
- data/ext/fast_mmaped_file/jsmn.c +259 -243
- data/ext/fast_mmaped_file/mmap.c +332 -0
- data/ext/fast_mmaped_file/mmap.h +9 -0
- data/ext/fast_mmaped_file/rendering.c +110 -37
- data/ext/fast_mmaped_file/rendering.h +2 -2
- data/ext/fast_mmaped_file/utils.c +3 -2
- data/ext/fast_mmaped_file/utils.h +1 -1
- data/ext/fast_mmaped_file/value_access.c +88 -48
- data/ext/fast_mmaped_file/value_access.h +4 -2
- data/lib/fast_mmaped_file.bundle +0 -0
- data/lib/prometheus/client/helper/entry_parser.rb +1 -1
- data/lib/prometheus/client/helper/mmaped_file.rb +4 -6
- data/lib/prometheus/client/helper/plain_file.rb +3 -3
- data/lib/prometheus/client/mmaped_dict.rb +2 -7
- data/lib/prometheus/client/simple_value.rb +0 -1
- data/lib/prometheus/client/support/unicorn.rb +9 -0
- data/lib/prometheus/client/version.rb +1 -1
- metadata +3 -25
- data/ext/fast_mmaped_file/hashmap.h +0 -266
- data/ext/fast_mmaped_file/jsmn.h +0 -76
- data/lib/prometheus/client/version.rb.orig +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e546160d5f4baf942cd59595ba9f4738f5bb112a
|
4
|
+
data.tar.gz: d2369f52a14a938fec91c09d372396c031992234
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c02fd8414f65779692b0aac43e89f53a87492843bae04891924838320533f495de9977ba2c99c72d34553d937268fa0ee6b5859861f1f13a86a5969c74b4320f
|
7
|
+
data.tar.gz: e9bf20cc0e44dd7307b429dd088d82702a25073bbd72ff0cc9315d4e2580e7438042b98032c21f7ddb0f668afd6bf42c0018e3f4bd81dfb40c30b79d0ecab9b4
|
@@ -1,7 +1,22 @@
|
|
1
1
|
require 'mkmf'
|
2
|
+
require 'fileutils'
|
3
|
+
|
2
4
|
$CFLAGS << ' -std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra -Werror'
|
3
5
|
|
6
|
+
if enable_config('address-sanitizer')
|
7
|
+
$CFLAGS << ' -O -fsanitize=address -fno-omit-frame-pointer -g'
|
8
|
+
end
|
9
|
+
|
4
10
|
CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
|
5
11
|
|
12
|
+
cwd = File.expand_path(File.dirname(__FILE__))
|
13
|
+
vendor_dir = File.join(cwd, '../../vendor/c')
|
14
|
+
src_dir = File.join(cwd, '../../ext/fast_mmaped_file')
|
15
|
+
|
16
|
+
src_files = %W[#{vendor_dir}/jsmn/jsmn.c #{vendor_dir}/hashmap/src/hashmap.c]
|
17
|
+
FileUtils.cp(src_files, src_dir)
|
18
|
+
|
19
|
+
$INCFLAGS << " -I#{vendor_dir}/jsmn -I#{vendor_dir}/hashmap/src"
|
20
|
+
|
6
21
|
dir_config('fast_mmaped_file')
|
7
22
|
create_makefile('fast_mmaped_file')
|
@@ -1,16 +1,20 @@
|
|
1
|
+
#include <errno.h>
|
1
2
|
#include <ruby.h>
|
2
3
|
#include <ruby/intern.h>
|
3
|
-
#include <errno.h>
|
4
4
|
|
5
|
-
#include <
|
6
|
-
#include <value_access.h>
|
7
|
-
#include <globals.h>
|
5
|
+
#include <sys/mman.h>
|
8
6
|
|
9
|
-
#include <jsmn.h>
|
10
7
|
#include <hashmap.h>
|
11
|
-
#include <
|
12
|
-
|
13
|
-
#include
|
8
|
+
#include <jsmn.h>
|
9
|
+
|
10
|
+
#include "globals.h"
|
11
|
+
#include "utils.h"
|
12
|
+
#include "value_access.h"
|
13
|
+
|
14
|
+
#include "file_parsing.h"
|
15
|
+
#include "file_reading.h"
|
16
|
+
#include "mmap.h"
|
17
|
+
#include "rendering.h"
|
14
18
|
|
15
19
|
VALUE MMAPED_FILE = Qnil;
|
16
20
|
|
@@ -21,9 +25,13 @@ ID sym_gauge;
|
|
21
25
|
ID sym_pid;
|
22
26
|
ID sym_samples;
|
23
27
|
|
24
|
-
|
25
|
-
|
26
|
-
|
28
|
+
VALUE prom_eParsingError;
|
29
|
+
|
30
|
+
int aggregate_files(struct hashmap *map, VALUE list_of_files) {
|
31
|
+
buffer_t reading_buffer;
|
32
|
+
memset(&reading_buffer, 0, sizeof(buffer_t));
|
33
|
+
|
34
|
+
for (int i = 0; i < RARRAY_LEN(list_of_files); i++) {
|
27
35
|
VALUE params = RARRAY_PTR(list_of_files)[i];
|
28
36
|
file_t file;
|
29
37
|
|
@@ -32,19 +40,19 @@ int aggregate_files(struct hashmap *map, VALUE list_of_files){
|
|
32
40
|
return 0;
|
33
41
|
}
|
34
42
|
|
35
|
-
if (!read_from_file(&file, &reading_buffer)){
|
43
|
+
if (!read_from_file(&file, &reading_buffer)) {
|
36
44
|
buffer_dispose(&reading_buffer);
|
37
45
|
file_close(&file);
|
38
46
|
return 0;
|
39
47
|
}
|
40
48
|
|
41
|
-
if (!process_buffer(&file, &reading_buffer, map)){
|
49
|
+
if (!process_buffer(&file, &reading_buffer, map)) {
|
42
50
|
buffer_dispose(&reading_buffer);
|
43
51
|
file_close(&file);
|
44
52
|
return 0;
|
45
53
|
}
|
46
54
|
|
47
|
-
if (!file_close(&file)){
|
55
|
+
if (!file_close(&file)) {
|
48
56
|
buffer_dispose(&reading_buffer);
|
49
57
|
return 0;
|
50
58
|
}
|
@@ -54,30 +62,36 @@ int aggregate_files(struct hashmap *map, VALUE list_of_files){
|
|
54
62
|
return 1;
|
55
63
|
}
|
56
64
|
|
57
|
-
VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list){
|
65
|
+
VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list) {
|
58
66
|
struct hashmap map;
|
59
67
|
hashmap_setup(&map);
|
60
68
|
|
61
|
-
if (!aggregate_files(&map, file_list)){
|
69
|
+
if (!aggregate_files(&map, file_list)) { // all entries in map are now copies that need to be disposed
|
62
70
|
hashmap_destroy(&map);
|
63
71
|
raise_last_exception();
|
64
72
|
return Qnil;
|
65
73
|
}
|
66
74
|
|
67
|
-
|
75
|
+
entry_t **sorted_entries;
|
68
76
|
|
69
|
-
if (!sort_map_entries(&map, &sorted_entries)){
|
70
|
-
entries_destroy(&map);
|
77
|
+
if (!sort_map_entries(&map, &sorted_entries)) {
|
71
78
|
hashmap_destroy(&map);
|
72
79
|
|
73
80
|
raise_last_exception();
|
74
81
|
return Qnil;
|
75
82
|
}
|
76
83
|
|
77
|
-
VALUE rv =
|
84
|
+
VALUE rv = rb_str_new("", 0);
|
85
|
+
if (!entries_to_string(rv, sorted_entries, hashmap_size(&map))) {
|
86
|
+
free(sorted_entries);
|
87
|
+
hashmap_destroy(&map);
|
88
|
+
|
89
|
+
raise_last_exception();
|
90
|
+
return Qnil;
|
91
|
+
}
|
78
92
|
|
93
|
+
RB_GC_GUARD(file_list); // ensure file list is not GCed before this point
|
79
94
|
free(sorted_entries);
|
80
|
-
entries_destroy(&map);
|
81
95
|
hashmap_destroy(&map);
|
82
96
|
return rv;
|
83
97
|
}
|
@@ -90,11 +104,22 @@ void Init_fast_mmaped_file() {
|
|
90
104
|
sym_pid = rb_intern("pid");
|
91
105
|
sym_samples = rb_intern("samples");
|
92
106
|
|
93
|
-
|
107
|
+
prom_eParsingError = rb_define_class("PrometheusParsingError", rb_eRuntimeError);
|
108
|
+
|
109
|
+
MMAPED_FILE = rb_define_class("FastMmapedFile", rb_cObject);
|
110
|
+
rb_define_const(MMAPED_FILE, "MAP_SHARED", INT2FIX(MAP_SHARED));
|
111
|
+
|
94
112
|
rb_define_singleton_method(MMAPED_FILE, "to_metrics", method_to_metrics, 1);
|
95
113
|
|
96
|
-
|
114
|
+
rb_define_alloc_func(MMAPED_FILE, mm_s_alloc);
|
115
|
+
rb_define_singleton_method(MMAPED_FILE, "new", mm_s_new, -1);
|
116
|
+
rb_define_method(MMAPED_FILE, "initialize", mm_init, 1);
|
117
|
+
rb_define_method(MMAPED_FILE, "slice", mm_aref_m, -1);
|
118
|
+
rb_define_method(MMAPED_FILE, "sync", mm_msync, -1);
|
119
|
+
rb_define_method(MMAPED_FILE, "munmap", mm_unmap, 0);
|
120
|
+
|
97
121
|
rb_define_method(MMAPED_FILE, "used", method_load_used, 0);
|
98
122
|
rb_define_method(MMAPED_FILE, "used=", method_save_used, 1);
|
99
|
-
rb_define_method(MMAPED_FILE, "
|
123
|
+
rb_define_method(MMAPED_FILE, "fetch_entry", method_fetch_entry, 3);
|
124
|
+
rb_define_method(MMAPED_FILE, "upsert_entry", method_upsert_entry, 3);
|
100
125
|
}
|
@@ -1,67 +1,38 @@
|
|
1
1
|
#include <hashmap.h>
|
2
|
-
#include <file_format.h>
|
3
|
-
#include <globals.h>
|
4
|
-
#include <utils.h>
|
5
2
|
#include <jsmn.h>
|
6
|
-
#include
|
3
|
+
#include <ruby.h>
|
7
4
|
|
8
|
-
|
5
|
+
#include "file_format.h"
|
6
|
+
#include "file_parsing.h"
|
7
|
+
#include "globals.h"
|
8
|
+
#include "utils.h"
|
9
9
|
|
10
|
-
|
10
|
+
HASHMAP_FUNCS_CREATE(entry, const entry_t, entry_t)
|
11
11
|
|
12
|
-
|
13
|
-
size_t hash = 0;
|
12
|
+
typedef int (*compare_fn)(const void *a, const void *b);
|
14
13
|
|
15
|
-
|
16
|
-
hash += *(const char *)(entry->json + i);
|
17
|
-
hash += (hash << 10);
|
18
|
-
hash ^= (hash >> 6);
|
19
|
-
}
|
20
|
-
hash += (hash << 3);
|
21
|
-
hash ^= (hash >> 11);
|
22
|
-
hash += (hash << 15);
|
23
|
-
return hash;
|
24
|
-
}
|
14
|
+
static size_t hashmap_hash_entry(const entry_t *entry) { return hashmap_hash_string(entry->json); }
|
25
15
|
|
26
|
-
int hashmap_compare_entry(const
|
27
|
-
{
|
28
|
-
if (a->json_size != b->json_size){
|
16
|
+
static int hashmap_compare_entry(const entry_t *a, const entry_t *b) {
|
17
|
+
if (a->json_size != b->json_size) {
|
29
18
|
return -1;
|
30
19
|
}
|
31
20
|
|
32
|
-
if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)){
|
21
|
+
if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)) {
|
33
22
|
return -1;
|
34
23
|
}
|
35
24
|
|
36
|
-
|
25
|
+
return strncmp(a->json, b->json, a->json_size);
|
37
26
|
}
|
38
27
|
|
39
|
-
|
40
|
-
entry_struct *copied = (entry_struct *)malloc(sizeof(entry_struct));
|
41
|
-
if (copied == NULL) {
|
42
|
-
return NULL;
|
43
|
-
}
|
44
|
-
memcpy(copied, entry, sizeof(entry_struct));
|
45
|
-
|
46
|
-
copied->json = malloc(entry->json_size);
|
47
|
-
if (copied->json == NULL){
|
48
|
-
free(copied);
|
49
|
-
return NULL;
|
50
|
-
}
|
51
|
-
|
52
|
-
memcpy(copied->json, entry->json, entry->json_size);
|
53
|
-
|
54
|
-
return copied;
|
55
|
-
}
|
56
|
-
|
57
|
-
void entry_free(entry_struct *entry){
|
28
|
+
static void entry_free(entry_t *entry) {
|
58
29
|
free(entry->json);
|
59
30
|
free(entry);
|
60
31
|
}
|
61
32
|
|
62
|
-
void merge_entry(
|
63
|
-
if (entry->type == sym_gauge){
|
64
|
-
if (entry->multiprocess_mode == sym_min){
|
33
|
+
static void merge_entry(entry_t *found, const entry_t *entry) {
|
34
|
+
if (entry->type == sym_gauge) {
|
35
|
+
if (entry->multiprocess_mode == sym_min) {
|
65
36
|
found->value = min(found->value, entry->value);
|
66
37
|
} else if (entry->multiprocess_mode == sym_max) {
|
67
38
|
found->value = max(found->value, entry->value);
|
@@ -75,60 +46,72 @@ void merge_entry(entry_struct *found, const entry_struct *entry){
|
|
75
46
|
}
|
76
47
|
}
|
77
48
|
|
78
|
-
|
79
|
-
|
80
|
-
if (found){
|
49
|
+
void merge_or_store(struct hashmap *map, entry_t *entry) {
|
50
|
+
entry_t *found = entry_hashmap_get(map, entry);
|
51
|
+
if (found) {
|
81
52
|
merge_entry(found, entry);
|
53
|
+
entry_free(entry);
|
82
54
|
} else {
|
83
|
-
|
84
|
-
if (copy == NULL) {
|
85
|
-
save_exception(rb_eNoMemError, "Failed copying metrics entry");
|
86
|
-
return 0;
|
87
|
-
}
|
88
|
-
entry_hashmap_put(map, copy, copy); // use the hashmap like hashset actually
|
55
|
+
entry_hashmap_put(map, entry, entry); // use the hashmap like hashset actually
|
89
56
|
}
|
90
|
-
return 1;
|
91
57
|
}
|
92
58
|
|
93
|
-
inline
|
94
|
-
|
59
|
+
inline entry_t *entry_new(buffer_t *source, uint32_t pos, uint32_t encoded_len, file_t *file_info) {
|
60
|
+
entry_t *entry = calloc(1, sizeof(entry_t));
|
61
|
+
if (entry == NULL) {
|
62
|
+
return NULL;
|
63
|
+
}
|
64
|
+
|
65
|
+
entry->json = malloc(encoded_len + 1);
|
66
|
+
if (entry->json == NULL) {
|
67
|
+
free(entry);
|
68
|
+
return NULL;
|
69
|
+
}
|
70
|
+
|
71
|
+
memcpy(entry->json, source->buffer + pos, encoded_len);
|
72
|
+
entry->json[encoded_len] = '\0';
|
73
|
+
entry->json_size = encoded_len;
|
74
|
+
|
75
|
+
entry->pid = file_info->pid;
|
76
|
+
entry->multiprocess_mode = file_info->multiprocess_mode;
|
77
|
+
entry->type = file_info->type;
|
78
|
+
|
79
|
+
char *value_ptr = source->buffer + pos + encoded_len + padding_length(encoded_len);
|
80
|
+
memcpy(&(entry->value), value_ptr, sizeof(double));
|
81
|
+
|
82
|
+
return entry;
|
95
83
|
}
|
96
84
|
|
97
|
-
|
85
|
+
static int add_parsed_name(entry_t *entry) {
|
98
86
|
jsmn_parser parser;
|
99
87
|
jsmn_init(&parser);
|
100
88
|
|
101
|
-
jsmntok_t
|
102
|
-
|
89
|
+
jsmntok_t tokens[2];
|
90
|
+
memset(&tokens, 0, sizeof(tokens));
|
103
91
|
|
104
|
-
|
105
|
-
|
106
|
-
}
|
92
|
+
jsmn_parse(&parser, entry->json, entry->json_size, tokens, 2);
|
93
|
+
jsmntok_t *name_token = &tokens[1];
|
107
94
|
|
108
|
-
|
109
|
-
|
95
|
+
if (name_token->start < name_token->end && name_token->start > 0) {
|
96
|
+
entry->name = entry->json + name_token->start;
|
97
|
+
entry->name_len = name_token->end - name_token->start;
|
98
|
+
return 1;
|
99
|
+
}
|
100
|
+
return 0;
|
110
101
|
}
|
111
102
|
|
112
|
-
int entry_lexical_comparator(const
|
103
|
+
static int entry_lexical_comparator(const entry_t **a, const entry_t **b) {
|
113
104
|
size_t min_length = min((*a)->json_size, (*b)->json_size);
|
114
105
|
return strncmp((*a)->json, (*b)->json, min_length);
|
115
106
|
}
|
116
107
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
}
|
122
|
-
|
123
|
-
void entries_destroy(struct hashmap *map){
|
124
|
-
struct hashmap_iter *iter;
|
125
|
-
for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
|
126
|
-
entry_struct *entry = (entry_struct *)entry_hashmap_iter_get_key(iter);
|
127
|
-
entry_free(entry);
|
128
|
-
}
|
108
|
+
void hashmap_setup(struct hashmap *map) {
|
109
|
+
hashmap_init(map, (size_t(*)(const void *))hashmap_hash_entry,
|
110
|
+
(int (*)(const void *, const void *))hashmap_compare_entry, 1000);
|
111
|
+
hashmap_set_key_alloc_funcs(map, NULL, (void (*)(void *))entry_free);
|
129
112
|
}
|
130
113
|
|
131
|
-
int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map){
|
114
|
+
int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map) {
|
132
115
|
if (source->size < START_POSITION) {
|
133
116
|
// nothing to read
|
134
117
|
return 1;
|
@@ -136,10 +119,9 @@ int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map){
|
|
136
119
|
uint32_t used;
|
137
120
|
memcpy(&used, source->buffer, sizeof(uint32_t));
|
138
121
|
|
139
|
-
if (used > source->size){
|
140
|
-
|
141
|
-
|
142
|
-
file_info->path, used, source->size);
|
122
|
+
if (used > source->size) {
|
123
|
+
save_exception(prom_eParsingError, "source file %s corrupted, used %u > file size %u", file_info->path, used,
|
124
|
+
source->size);
|
143
125
|
return 0;
|
144
126
|
}
|
145
127
|
|
@@ -152,62 +134,55 @@ int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map){
|
|
152
134
|
uint32_t value_offset = encoded_len + padding_length(encoded_len);
|
153
135
|
|
154
136
|
if (pos + value_offset + sizeof(double) > used) {
|
155
|
-
|
156
|
-
|
157
|
-
file_info->path, used, pos + value_offset + sizeof(double));
|
137
|
+
save_exception(prom_eParsingError, "source file %s corrupted, used %u < stored data length %u",
|
138
|
+
file_info->path, used, pos + value_offset + sizeof(double));
|
158
139
|
return 0;
|
159
140
|
}
|
160
141
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
entry.pid = file_info->pid;
|
165
|
-
entry.type = file_info->type;
|
166
|
-
|
167
|
-
memcpy(&entry.value, entry.json + value_offset, sizeof(double));
|
168
|
-
|
169
|
-
if (!process_entry(map, &entry)) {
|
170
|
-
entries_destroy(map);
|
142
|
+
entry_t *entry = entry_new(source, pos, encoded_len, file_info);
|
143
|
+
if (entry == NULL) {
|
144
|
+
save_exception(rb_eNoMemError, "Failed creating metrics entry");
|
171
145
|
return 0;
|
172
146
|
}
|
173
147
|
|
148
|
+
merge_or_store(map, entry);
|
149
|
+
|
174
150
|
pos += value_offset + sizeof(double);
|
175
151
|
}
|
176
152
|
return 1;
|
177
153
|
}
|
178
154
|
|
179
|
-
int sort_map_entries(const struct hashmap *map,
|
155
|
+
int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries) {
|
180
156
|
size_t num = hashmap_size(map);
|
181
157
|
|
182
|
-
|
183
|
-
entry_struct **list = malloc(list_size);
|
158
|
+
entry_t **list = calloc(num, sizeof(entry_t *));
|
184
159
|
|
185
|
-
if (list == NULL){
|
186
|
-
save_exception(rb_eNoMemError, "Couldn't allocate %zu memory",
|
160
|
+
if (list == NULL) {
|
161
|
+
save_exception(rb_eNoMemError, "Couldn't allocate for %zu memory", num * sizeof(entry_t *));
|
187
162
|
return 0;
|
188
163
|
}
|
189
164
|
|
190
165
|
size_t cnt = 0;
|
191
166
|
struct hashmap_iter *iter;
|
192
167
|
for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
|
193
|
-
|
194
|
-
add_parsed_name(entry)
|
195
|
-
|
196
|
-
|
197
|
-
|
168
|
+
entry_t *entry = (entry_t *)entry_hashmap_iter_get_key(iter);
|
169
|
+
if (add_parsed_name(entry)) {
|
170
|
+
list[cnt] = entry;
|
171
|
+
cnt++;
|
172
|
+
}
|
198
173
|
}
|
199
|
-
if (cnt != num){
|
174
|
+
if (cnt != num) {
|
200
175
|
save_exception(rb_eRuntimeError, "Processed entries %zu != map entries %zu", cnt, num);
|
201
176
|
free(list);
|
202
177
|
return 0;
|
203
178
|
}
|
204
179
|
|
205
|
-
qsort(list, cnt, sizeof(
|
180
|
+
qsort(list, cnt, sizeof(entry_t *), (compare_fn)&entry_lexical_comparator);
|
206
181
|
*sorted_entries = list;
|
207
182
|
return 1;
|
208
183
|
}
|
209
184
|
|
210
|
-
int is_pid_significant(const
|
185
|
+
int is_pid_significant(const entry_t *e) {
|
211
186
|
ID mp = e->multiprocess_mode;
|
212
187
|
return e->type == sym_gauge && !(mp == sym_min || mp == sym_max || mp == sym_livesum);
|
213
188
|
}
|