prometheus-client-mmap 0.7.0.beta41 → 0.7.0.beta42

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49ac78e6bb20f940989a545386f5a8340dd3a046
-  data.tar.gz: e11aeb1946d4c4afa0f09e5037bbb7b7e5c23a6e
+  metadata.gz: e546160d5f4baf942cd59595ba9f4738f5bb112a
+  data.tar.gz: d2369f52a14a938fec91c09d372396c031992234
 SHA512:
-  metadata.gz: 961b21816d5e183e2ea030512481b9627671d66b632ed2be9644eda9ed512a933a4f96a6fef06163ea21998675c6a226e6571f3bda4a10b97c83189d066f489c
-  data.tar.gz: 2351c759ee985a918c5aecaa752cb4071fb0686f4185944ea13559b9bc3cb1fd7b8b8c2e1a166be689f0a960a02204d6a277243c331ddde97814f1a393f632c5
+  metadata.gz: c02fd8414f65779692b0aac43e89f53a87492843bae04891924838320533f495de9977ba2c99c72d34553d937268fa0ee6b5859861f1f13a86a5969c74b4320f
+  data.tar.gz: e9bf20cc0e44dd7307b429dd088d82702a25073bbd72ff0cc9315d4e2580e7438042b98032c21f7ddb0f668afd6bf42c0018e3f4bd81dfb40c30b79d0ecab9b4

data/ext/fast_mmaped_file/extconf.rb

@@ -1,7 +1,22 @@
 require 'mkmf'
+require 'fileutils'
+
 $CFLAGS << ' -std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra -Werror'
 
+if enable_config('address-sanitizer')
+  $CFLAGS << ' -O -fsanitize=address -fno-omit-frame-pointer -g'
+end
+
 CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
 
+cwd = File.expand_path(File.dirname(__FILE__))
+vendor_dir = File.join(cwd, '../../vendor/c')
+src_dir = File.join(cwd, '../../ext/fast_mmaped_file')
+
+src_files = %W[#{vendor_dir}/jsmn/jsmn.c #{vendor_dir}/hashmap/src/hashmap.c]
+FileUtils.cp(src_files, src_dir)
+
+$INCFLAGS << " -I#{vendor_dir}/jsmn -I#{vendor_dir}/hashmap/src"
+
 dir_config('fast_mmaped_file')
 create_makefile('fast_mmaped_file')

data/ext/fast_mmaped_file/fast_mmaped_file.c

@@ -1,16 +1,20 @@
+#include <errno.h>
 #include <ruby.h>
 #include <ruby/intern.h>
-#include <errno.h>
 
-#include <utils.h>
-#include <value_access.h>
-#include <globals.h>
+#include <sys/mman.h>
 
-#include <jsmn.h>
 #include <hashmap.h>
-#include <file_reading.h>
-#include <file_parsing.h>
-#include <rendering.h>
+#include <jsmn.h>
+
+#include "globals.h"
+#include "utils.h"
+#include "value_access.h"
+
+#include "file_parsing.h"
+#include "file_reading.h"
+#include "mmap.h"
+#include "rendering.h"
 
 VALUE MMAPED_FILE = Qnil;
 
@@ -21,9 +25,13 @@ ID sym_gauge;
 ID sym_pid;
 ID sym_samples;
 
-int aggregate_files(struct hashmap *map, VALUE list_of_files){
-    buffer_t reading_buffer = (buffer_t) { .buffer = NULL };
-    for(int i =0; i< RARRAY_LEN(list_of_files); i++){
+VALUE prom_eParsingError;
+
+int aggregate_files(struct hashmap *map, VALUE list_of_files) {
+    buffer_t reading_buffer;
+    memset(&reading_buffer, 0, sizeof(buffer_t));
+
+    for (int i = 0; i < RARRAY_LEN(list_of_files); i++) {
         VALUE params = RARRAY_PTR(list_of_files)[i];
         file_t file;
 
@@ -32,19 +40,19 @@ int aggregate_files(struct hashmap *map, VALUE list_of_files){
             return 0;
         }
 
-        if (!read_from_file(&file, &reading_buffer)){
+        if (!read_from_file(&file, &reading_buffer)) {
             buffer_dispose(&reading_buffer);
             file_close(&file);
             return 0;
         }
 
-        if (!process_buffer(&file, &reading_buffer, map)){
+        if (!process_buffer(&file, &reading_buffer, map)) {
             buffer_dispose(&reading_buffer);
             file_close(&file);
             return 0;
         }
 
-        if (!file_close(&file)){
+        if (!file_close(&file)) {
             buffer_dispose(&reading_buffer);
             return 0;
         }
@@ -54,30 +62,36 @@ int aggregate_files(struct hashmap *map, VALUE list_of_files){
     return 1;
 }
 
-VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list){
+VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list) {
     struct hashmap map;
     hashmap_setup(&map);
 
-    if (!aggregate_files(&map, file_list)){ // all entries in map are now copies that need to be disposed
+    if (!aggregate_files(&map, file_list)) {  // all entries in map are now copies that need to be disposed
         hashmap_destroy(&map);
         raise_last_exception();
         return Qnil;
     }
 
-    entry_struct **sorted_entries;
+    entry_t **sorted_entries;
 
-    if (!sort_map_entries(&map, &sorted_entries)){
-        entries_destroy(&map);
+    if (!sort_map_entries(&map, &sorted_entries)) {
         hashmap_destroy(&map);
 
         raise_last_exception();
         return Qnil;
     }
 
-    VALUE rv = entries_to_string(sorted_entries, hashmap_size(&map));
+    VALUE rv = rb_str_new("", 0);
+    if (!entries_to_string(rv, sorted_entries, hashmap_size(&map))) {
+        free(sorted_entries);
+        hashmap_destroy(&map);
+
+        raise_last_exception();
+        return Qnil;
+    }
 
+    RB_GC_GUARD(file_list);  // ensure file list is not GCed before this point
     free(sorted_entries);
-    entries_destroy(&map);
     hashmap_destroy(&map);
     return rv;
 }
@@ -90,11 +104,22 @@ void Init_fast_mmaped_file() {
     sym_pid = rb_intern("pid");
     sym_samples = rb_intern("samples");
 
-    MMAPED_FILE = rb_define_module("FastMmapedFile");
+    prom_eParsingError = rb_define_class("PrometheusParsingError", rb_eRuntimeError);
+
+    MMAPED_FILE = rb_define_class("FastMmapedFile", rb_cObject);
+    rb_define_const(MMAPED_FILE, "MAP_SHARED", INT2FIX(MAP_SHARED));
+
     rb_define_singleton_method(MMAPED_FILE, "to_metrics", method_to_metrics, 1);
 
-    rb_define_method(MMAPED_FILE, "get_double", method_get_double, 1);
+    rb_define_alloc_func(MMAPED_FILE, mm_s_alloc);
+    rb_define_singleton_method(MMAPED_FILE, "new", mm_s_new, -1);
+    rb_define_method(MMAPED_FILE, "initialize", mm_init, 1);
+    rb_define_method(MMAPED_FILE, "slice", mm_aref_m, -1);
+    rb_define_method(MMAPED_FILE, "sync", mm_msync, -1);
+    rb_define_method(MMAPED_FILE, "munmap", mm_unmap, 0);
+
     rb_define_method(MMAPED_FILE, "used", method_load_used, 0);
     rb_define_method(MMAPED_FILE, "used=", method_save_used, 1);
-    rb_define_method(MMAPED_FILE, "add_entry", method_add_entry, 3);
+    rb_define_method(MMAPED_FILE, "fetch_entry", method_fetch_entry, 3);
+    rb_define_method(MMAPED_FILE, "upsert_entry", method_upsert_entry, 3);
 }

data/ext/fast_mmaped_file/file_parsing.c

@@ -1,67 +1,38 @@
 #include <hashmap.h>
-#include <file_format.h>
-#include <globals.h>
-#include <utils.h>
 #include <jsmn.h>
-#include "file_parsing.h"
+#include <ruby.h>
 
-HASHMAP_FUNCS_CREATE(entry, const entry_struct, entry_struct)
+#include "file_format.h"
+#include "file_parsing.h"
+#include "globals.h"
+#include "utils.h"
 
-typedef int (*compare_fn) (const void *a, const void *b);
+HASHMAP_FUNCS_CREATE(entry, const entry_t, entry_t)
 
-size_t hashmap_hash_entry(const entry_struct *entry){
-	size_t hash = 0;
+typedef int (*compare_fn)(const void *a, const void *b);
 
-	for (size_t i = 0; i < entry->json_size; i++) {
-		hash += *(const char *)(entry->json + i);
-		hash += (hash << 10);
-		hash ^= (hash >> 6);
-	}
-	hash += (hash << 3);
-	hash ^= (hash >> 11);
-	hash += (hash << 15);
-	return hash;
-}
+static size_t hashmap_hash_entry(const entry_t *entry) { return hashmap_hash_string(entry->json); }
 
-int hashmap_compare_entry(const entry_struct *a, const entry_struct *b)
-{
-    if (a->json_size != b->json_size){
+static int hashmap_compare_entry(const entry_t *a, const entry_t *b) {
+    if (a->json_size != b->json_size) {
         return -1;
     }
 
-    if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)){ //OPTIONAL TODO: optimize rb out
+    if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)) {
         return -1;
     }
 
-	return strncmp(a->json, b->json, a->json_size);
+    return strncmp(a->json, b->json, a->json_size);
 }
 
-entry_struct *copy_entry(const entry_struct *entry){
-    entry_struct *copied = (entry_struct *)malloc(sizeof(entry_struct));
-    if (copied == NULL) {
-        return NULL;
-    }
-    memcpy(copied, entry, sizeof(entry_struct));
-
-    copied->json = malloc(entry->json_size);
-    if (copied->json == NULL){
-        free(copied);
-        return NULL;
-    }
-
-    memcpy(copied->json, entry->json, entry->json_size);
-
-    return copied;
-}
-
-void entry_free(entry_struct *entry){
+static void entry_free(entry_t *entry) {
     free(entry->json);
     free(entry);
 }
 
-void merge_entry(entry_struct *found, const entry_struct *entry){
-    if (entry->type == sym_gauge){
-        if (entry->multiprocess_mode == sym_min){
+static void merge_entry(entry_t *found, const entry_t *entry) {
+    if (entry->type == sym_gauge) {
+        if (entry->multiprocess_mode == sym_min) {
             found->value = min(found->value, entry->value);
         } else if (entry->multiprocess_mode == sym_max) {
             found->value = max(found->value, entry->value);
@@ -75,60 +46,72 @@ void merge_entry(entry_struct *found, const entry_struct *entry){
     }
 }
 
-int process_entry(struct hashmap *map, const entry_struct *entry){
-    entry_struct *found = entry_hashmap_get(map, entry);
-    if (found){
+void merge_or_store(struct hashmap *map, entry_t *entry) {
+    entry_t *found = entry_hashmap_get(map, entry);
+    if (found) {
         merge_entry(found, entry);
+        entry_free(entry);
     } else {
-        entry_struct *copy = copy_entry(entry);
-        if (copy == NULL) {
-            save_exception(rb_eNoMemError, "Failed copying metrics entry");
-            return 0;
-        }
-        entry_hashmap_put(map, copy, copy); // use the hashmap like hashset actually
+        entry_hashmap_put(map, entry, entry);  // use the hashmap like hashset actually
     }
-    return 1;
 }
 
-inline entry_struct entry_new(char* json, size_t json_size, double value, uint32_t pid){
-    return (entry_struct) { .json = json, .json_size = json_size, .pid = pid, .value = value, .name_len = 0 };
+inline entry_t *entry_new(buffer_t *source, uint32_t pos, uint32_t encoded_len, file_t *file_info) {
+    entry_t *entry = calloc(1, sizeof(entry_t));
+    if (entry == NULL) {
+        return NULL;
+    }
+
+    entry->json = malloc(encoded_len + 1);
+    if (entry->json == NULL) {
+        free(entry);
+        return NULL;
+    }
+
+    memcpy(entry->json, source->buffer + pos, encoded_len);
+    entry->json[encoded_len] = '\0';
+    entry->json_size = encoded_len;
+
+    entry->pid = file_info->pid;
+    entry->multiprocess_mode = file_info->multiprocess_mode;
+    entry->type = file_info->type;
+
+    char *value_ptr = source->buffer + pos + encoded_len + padding_length(encoded_len);
+    memcpy(&(entry->value), value_ptr, sizeof(double));
+
+    return entry;
 }
 
-void add_parsed_name(entry_struct *entry){
+static int add_parsed_name(entry_t *entry) {
     jsmn_parser parser;
     jsmn_init(&parser);
 
-    jsmntok_t t[2];
-    int r = jsmn_parse(&parser, entry->json, entry->json_size, t, 2);
+    jsmntok_t tokens[2];
+    memset(&tokens, 0, sizeof(tokens));
 
-    if (r < 1) {
-        //TODO: better handle this case
-    }
+    jsmn_parse(&parser, entry->json, entry->json_size, tokens, 2);
+    jsmntok_t *name_token = &tokens[1];
 
-    entry->name = entry->json + t[1].start;
-    entry->name_len = t[1].end - t[1].start;
+    if (name_token->start < name_token->end && name_token->start > 0) {
+        entry->name = entry->json + name_token->start;
+        entry->name_len = name_token->end - name_token->start;
+        return 1;
+    }
+    return 0;
 }
 
-int entry_lexical_comparator(const entry_struct **a, const entry_struct **b){
+static int entry_lexical_comparator(const entry_t **a, const entry_t **b) {
     size_t min_length = min((*a)->json_size, (*b)->json_size);
     return strncmp((*a)->json, (*b)->json, min_length);
 }
 
-// public functions
-
-void hashmap_setup(struct hashmap *map){
-  hashmap_init(map, (size_t (*)(const void *))hashmap_hash_entry, (int (*)(const void *, const void *))hashmap_compare_entry, 1000);
-}
-
-void entries_destroy(struct hashmap *map){
-    struct hashmap_iter *iter;
-    for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
-        entry_struct *entry = (entry_struct *)entry_hashmap_iter_get_key(iter);
-        entry_free(entry);
-    }
+void hashmap_setup(struct hashmap *map) {
+    hashmap_init(map, (size_t(*)(const void *))hashmap_hash_entry,
+                 (int (*)(const void *, const void *))hashmap_compare_entry, 1000);
+    hashmap_set_key_alloc_funcs(map, NULL, (void (*)(void *))entry_free);
 }
 
-int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map){
+int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map) {
     if (source->size < START_POSITION) {
         // nothing to read
         return 1;
@@ -136,10 +119,9 @@ int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map){
     uint32_t used;
     memcpy(&used, source->buffer, sizeof(uint32_t));
 
-    if (used > source->size){
-        //TODO: add custom parsing error
-        save_exception(rb_eRuntimeError, "source file %s corrupted, used %u > file size %u",
-            file_info->path, used, source->size);
+    if (used > source->size) {
+        save_exception(prom_eParsingError, "source file %s corrupted, used %u > file size %u", file_info->path, used,
+                       source->size);
         return 0;
     }
 
@@ -152,62 +134,55 @@ int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map){
         uint32_t value_offset = encoded_len + padding_length(encoded_len);
 
         if (pos + value_offset + sizeof(double) > used) {
-            //TODO: add custom parsing error
-            save_exception(rb_eRuntimeError, "source file %s corrupted, used %u < stored data length %u",
-                        file_info->path, used, pos + value_offset + sizeof(double));
+            save_exception(prom_eParsingError, "source file %s corrupted, used %u < stored data length %u",
+                           file_info->path, used, pos + value_offset + sizeof(double));
             return 0;
         }
 
-        entry_struct entry = entry_new(source->buffer + pos, encoded_len, 0, 0.0);
-
-        entry.multiprocess_mode = file_info->multiprocess_mode;
-        entry.pid = file_info->pid;
-        entry.type = file_info->type;
-
-        memcpy(&entry.value, entry.json + value_offset, sizeof(double));
-
-        if (!process_entry(map, &entry)) {
-            entries_destroy(map);
+        entry_t *entry = entry_new(source, pos, encoded_len, file_info);
+        if (entry == NULL) {
+            save_exception(rb_eNoMemError, "Failed creating metrics entry");
             return 0;
         }
 
+        merge_or_store(map, entry);
+
         pos += value_offset + sizeof(double);
     }
     return 1;
 }
 
-int sort_map_entries(const struct hashmap *map, entry_struct ***sorted_entries){
+int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries) {
     size_t num = hashmap_size(map);
 
-    size_t list_size = num * sizeof(entry_struct *);
-    entry_struct **list = malloc(list_size);
+    entry_t **list = calloc(num, sizeof(entry_t *));
 
-    if (list == NULL){
-        save_exception(rb_eNoMemError, "Couldn't allocate %zu memory", list_size);
+    if (list == NULL) {
+        save_exception(rb_eNoMemError, "Couldn't allocate for %zu memory", num * sizeof(entry_t *));
         return 0;
     }
 
     size_t cnt = 0;
     struct hashmap_iter *iter;
     for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
-        entry_struct *entry = (entry_struct *)entry_hashmap_iter_get_key(iter);
-        add_parsed_name(entry);
-
-        list[cnt] = entry;
-        cnt++;
+        entry_t *entry = (entry_t *)entry_hashmap_iter_get_key(iter);
+        if (add_parsed_name(entry)) {
+            list[cnt] = entry;
+            cnt++;
+        }
     }
-    if (cnt != num){
+    if (cnt != num) {
         save_exception(rb_eRuntimeError, "Processed entries %zu != map entries %zu", cnt, num);
         free(list);
         return 0;
     }
 
-    qsort(list, cnt, sizeof(entry_struct *), (compare_fn)&entry_lexical_comparator);
+    qsort(list, cnt, sizeof(entry_t *), (compare_fn)&entry_lexical_comparator);
     *sorted_entries = list;
     return 1;
 }
 
-int is_pid_significant(const entry_struct *e){
+int is_pid_significant(const entry_t *e) {
     ID mp = e->multiprocess_mode;
     return e->type == sym_gauge && !(mp == sym_min || mp == sym_max || mp == sym_livesum);
 }