prometheus-client-mmap 0.7.0.beta39 → 0.7.0.beta40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 588b2d270f9bf3e00435fcac7442db8e7d6adea7
-  data.tar.gz: c9a90cdf6392e9a772a7ee131d4e3391d98027c3
+  metadata.gz: 2f51d0d3d39230181b8a35905cac312c52dde03b
+  data.tar.gz: 1ec3f78ad1276fb225d73e31f1f0e91369c8afa2
 SHA512:
-  metadata.gz: 680f78fb4a00d297bf5afa78b45ab38b98c362ec44408de5af420b3674fe5859ae89449ec57b4105d10a4864ba5468e0f4e35b3e8847dd433d7398641488a50d
-  data.tar.gz: baa5985553a72cab9a157b21b5a42b3682c59ee742410068922221497e21af356434239491c0d230600825a1eb1781adce3c3d87d2d9d63345feb61c71ec4397
+  metadata.gz: bf88cde2f108d7b3ade4ae808e0be40256acfa39cc8df26a3ce6ed7e023113314907f777afee1e28ac252293f46815b5bc898b9ab35485ccc61cc387b1c49934
+  data.tar.gz: 2d47f29dba886af6f5d3d462754712074e3c7d3e11c1072f399fd9621055299ec16dd5baa5e18c765220e013171d8b21596c7f96593ceb94479773821cd1b3cc

data/ext/fast_mmaped_file/extconf.rb

@@ -1,5 +1,5 @@
 require 'mkmf'
-$CFLAGS << ' -std=c99 -Wall -Wextra -Werror'
+$CFLAGS << ' -std=c11 -Wall -Wextra -Werror'
 
 CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
 

data/ext/fast_mmaped_file/fast_mmaped_file.c

@@ -1,175 +1,101 @@
 #include <ruby.h>
 #include <ruby/intern.h>
-#include <sys/mman.h>
-#include <unistd.h>
-#include <fcntl.h>
 #include <errno.h>
 
-#include "mmap.h"
+#include <utils.h>
+#include <value_access.h>
+#include <globals.h>
 
-VALUE MMAPED_FILE = Qnil;
-
-#define START_POSITION 8
-#define INITIAL_SIZE (2 * sizeof(int32_t))
-
-int open_and_extend_file(mm_ipc *i_mm, size_t len) {
-    int fd;
-
-    if ((fd = open(i_mm->t->path, i_mm->t->smode)) == -1) {
-        rb_raise(rb_eArgError, "Can't open %s", i_mm->t->path);
-    }
-
-    if (lseek(fd, len - i_mm->t->len - 1, SEEK_END) == -1) {
-        close(fd);
-        rb_raise(rb_eIOError, "Can't lseek %zu", len - i_mm->t->len - 1);
-    }
-
-    if (write(fd, "\000", 1) != 1) {
-        close(fd);
-        rb_raise(rb_eIOError, "Can't extend %s", i_mm->t->path);
-    }
-
-    return fd;
-}
-
-void expand(mm_ipc *i_mm, size_t len) {
-    if (len < i_mm->t->len) {
-        rb_raise(rb_eArgError, "Can't reduce the size of mmap");
-    }
-
-    if (munmap(i_mm->t->addr, i_mm->t->len)) {
-        rb_raise(rb_eArgError, "munmap failed");
-    }
-
-    int fd = open_and_extend_file(i_mm, len);
-
-    i_mm->t->addr = mmap(0, len, i_mm->t->pmode, i_mm->t->vscope, fd, i_mm->t->offset);
-
-    if (i_mm->t->addr == MAP_FAILED) {
-        close(fd);
-        rb_raise(rb_eArgError, "mmap failed");
-    }
-
-    if (close(fd) == -1) {
-        rb_raise(rb_eArgError, "Can't close %s", i_mm->t->path);
-    }
-
-    if ((i_mm->t->flag & MM_LOCK) && mlock(i_mm->t->addr, len) == -1) {
-        rb_raise(rb_eArgError, "mlock(%d)", errno);
-    }
-    i_mm->t->len = len;
-    i_mm->t->real = len;
-}
-
-inline uint32_t padding_length(uint32_t key_length) {
-    return 8 - (sizeof(uint32_t) + key_length) % 8; // padding | 8 byte aligned
-}
-
-void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
-    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
-
-    char *pos = (char *)i_mm->t->addr + offset;
 
-    memcpy(pos, &key_length, sizeof(uint32_t));
-    pos += sizeof(uint32_t);
+#include <jsmn.h>
+#include <hashmap.h>
+#include <file_reading.h>
+#include <file_parsing.h>
+#include <rendering.h>
 
-    memmove(pos, StringValuePtr(key), key_length);
-    pos += key_length;
-
-    memset(pos, ' ', padding_length(key_length));
-    pos += padding_length(key_length);
-
-    double val = NUM2DBL(value);
-    memcpy(pos, &val, sizeof(double));
-}
-
-inline uint32_t load_used(mm_ipc *i_mm) {
-    uint32_t used = *((uint32_t *)i_mm->t->addr);
-
-    if (used == 0) {
-        used = START_POSITION;
-    }
-    return used;
-}
-
-inline void save_used(mm_ipc *i_mm, uint32_t used) {
-    *((uint32_t *)i_mm->t->addr) = used;
-}
-
-VALUE method_load_used(VALUE self) {
-    mm_ipc *i_mm;
-
-    GET_MMAP(self, i_mm, MM_MODIFY);
-    return UINT2NUM(load_used(i_mm));
-}
-
-VALUE method_save_used(VALUE self, VALUE value) {
-    Check_Type(value, T_FIXNUM);
-    mm_ipc *i_mm;
-
-    GET_MMAP(self, i_mm, MM_MODIFY);
+VALUE MMAPED_FILE = Qnil;
 
-    if (i_mm->t->len < INITIAL_SIZE) {
-        expand(i_mm, INITIAL_SIZE);
+ID sym_min;
+ID sym_max;
+ID sym_livesum;
+ID sym_gauge;
+ID sym_pid;
+ID sym_samples;
+ID sym_key_name;
+
+int aggregate_files(struct hashmap *map, VALUE list_of_files){
+    buffer_t reading_buffer = (buffer_t) { .buffer = NULL };
+    for(int i =0; i< RARRAY_LEN(list_of_files); i++){
+        VALUE params = RARRAY_PTR(list_of_files)[i];
+        file_t file;
+
+        if (!file_open_from_params(&file, params)) {
+            buffer_dispose(&reading_buffer);
+            return 0;
+        }
+
+        if (!read_from_file(&file, &reading_buffer)){
+            buffer_dispose(&reading_buffer);
+            file_close(&file);
+            return 0;
+        }
+
+        if (!process_buffer(&file, &reading_buffer, map)){
+            buffer_dispose(&reading_buffer);
+            file_close(&file);
+            return 0;
+        }
+
+        if (!file_close(&file)){
+            buffer_dispose(&reading_buffer);
+            return 0;
+        }
     }
 
-    save_used(i_mm, NUM2UINT(value));
-    return value;
+    buffer_dispose(&reading_buffer);
+    return 1;
 }
 
-VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
-    Check_Type(positions, T_HASH);
-    Check_Type(key, T_STRING);
+VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list){
+    struct hashmap map;
+    hashmap_setup(&map);
 
-    VALUE position = rb_hash_lookup(positions, key);
-    if (position != Qnil){
-        return position;
+    if (!aggregate_files(&map, file_list)){ // all entries in map are now copies that need to be disposed
+        hashmap_destroy(&map);
+        raise_last_exception();
+        return Qnil;
     }
 
-    mm_ipc *i_mm;
-    GET_MMAP(self, i_mm, MM_MODIFY);
+    entry_struct **sorted_entries;
 
-    if (i_mm->t->flag & MM_FROZEN) {
-        rb_error_frozen("mmap");
-    }
+    if (!sort_map_entries(&map, &sorted_entries)){
+        entries_destroy(&map);
+        hashmap_destroy(&map);
 
-    if (RSTRING_LEN(key) > INT32_MAX) {
-        rb_raise(rb_eArgError, "string length gt %d", INT32_MAX);
+        raise_last_exception();
+        return Qnil;
     }
 
-    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
-    uint32_t value_offset = sizeof(uint32_t) + key_length + padding_length(key_length);
-    uint32_t entry_length = value_offset + sizeof(double);
-
-    uint32_t used = load_used(i_mm);
-    while (i_mm->t->len < (used + entry_length)) {
-        expand(i_mm, i_mm->t->len * 2);
-    }
+    VALUE rv = entries_to_string(sorted_entries, hashmap_size(&map));
 
-    save_entry(i_mm, used, key, value);
-    save_used(i_mm, used + entry_length);
-    return rb_hash_aset(positions, key, INT2NUM(used + value_offset));
-}
-
-VALUE method_get_double(VALUE self, VALUE index) {
-    mm_ipc *i_mm;
-    GET_MMAP(self, i_mm, MM_MODIFY);
-
-    Check_Type(index, T_FIXNUM);
-    size_t idx = NUM2UINT(index);
-
-    if ((i_mm->t->real + sizeof(double)) <= idx) {
-        rb_raise(rb_eIndexError, "index %zu out of string", idx);
-    }
-
-    double tmp;
-
-    memcpy(&tmp, (char *)i_mm->t->addr + idx, sizeof(double));
-    return DBL2NUM(tmp);
+    free(sorted_entries);
+    entries_destroy(&map);
+    hashmap_destroy(&map);
+    return rv;
 }
 
 void Init_fast_mmaped_file() {
+    sym_gauge = rb_intern("gauge");
+    sym_min = rb_intern("min");
+    sym_max = rb_intern("max");
+    sym_livesum = rb_intern("livesum");
+    sym_pid = rb_intern("pid");
+    sym_samples = rb_intern("samples");
+    sym_key_name = rb_intern("__key_name");
+
     MMAPED_FILE = rb_define_module("FastMmapedFile");
+    rb_define_singleton_method(MMAPED_FILE, "to_metrics", method_to_metrics, 1);
+
     rb_define_method(MMAPED_FILE, "get_double", method_get_double, 1);
     rb_define_method(MMAPED_FILE, "used", method_load_used, 0);
     rb_define_method(MMAPED_FILE, "used=", method_save_used, 1);

data/ext/fast_mmaped_file/file_format.h

@@ -0,0 +1,13 @@
+#ifndef FILE_FORMAT_H
+#define FILE_FORMAT_H
+
+#include <stdint.h>
+
+#define START_POSITION 8
+#define INITIAL_SIZE (2 * sizeof(int32_t))
+
+inline uint32_t padding_length(uint32_t key_length) {
+    return 8 - (sizeof(uint32_t) + key_length) % 8; // padding | 8 byte aligned
+}
+
+#endif

data/ext/fast_mmaped_file/file_parsing.c

@@ -0,0 +1,222 @@
+#include <hashmap.h>
+#include <file_format.h>
+#include <globals.h>
+#include <utils.h>
+#include <jsmn.h>
+#include "file_parsing.h"
+
+//ID sym_gauge;
+//ID sym_min;
+//ID sym_max;
+//ID sym_livesum;
+
+HASHMAP_FUNCS_CREATE(entry, const entry_struct, entry_struct)
+
+typedef int (*compare_fn) (const void *a, const void *b);
+
+inline int is_pid_significant(const entry_struct *a){
+    ID mp = a->multiprocess_mode;
+    return a->type == sym_gauge && !(mp == sym_min || mp == sym_max || mp == sym_livesum);
+}
+
+size_t hashmap_hash_entry(const entry_struct *entry){
+	size_t hash = 0;
+
+	for (size_t i = 0; i < entry->json_size; i++) {
+		hash += *(const char *)(entry->json + i);
+		hash += (hash << 10);
+		hash ^= (hash >> 6);
+	}
+	hash += (hash << 3);
+	hash ^= (hash >> 11);
+	hash += (hash << 15);
+	return hash;
+}
+
+int hashmap_compare_entry(const entry_struct *a, const entry_struct *b)
+{
+    if (a->json_size != b->json_size){
+        return -1;
+    }
+
+    if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)){ //OPTIONAL TODO: optimize rb out
+        return -1;
+    }
+
+	return strncmp(a->json, b->json, a->json_size);
+}
+
+entry_struct *copy_entry(const entry_struct *entry){
+    entry_struct *copied = (entry_struct *)malloc(sizeof(entry_struct));
+    if (copied == NULL) {
+        return NULL;
+    }
+    memcpy(copied, entry, sizeof(entry_struct));
+
+    copied->json = malloc(entry->json_size);
+    if (copied->json == NULL){
+        free(copied);
+        return NULL;
+    }
+
+    memcpy(copied->json, entry->json, entry->json_size);
+
+    return copied;
+}
+
+void entry_free(entry_struct *entry){
+    free(entry->json);
+    free(entry);
+}
+
+void merge_entry(entry_struct *found, const entry_struct *entry){
+    if (entry->type == sym_gauge){
+        if (entry->multiprocess_mode == sym_min){
+            found->value = min(found->value, entry->value);
+        } else if (entry->multiprocess_mode == sym_max) {
+            found->value = max(found->value, entry->value);
+        } else if (entry->multiprocess_mode == sym_livesum) {
+            found->value += entry->value;
+        } else {
+            found->value += entry->value; //TODO: this shouldn't happen
+        }
+    } else {
+        found->value += entry->value;
+    }
+}
+
+int process_entry(struct hashmap *map, const entry_struct *entry){
+    entry_struct *found = entry_hashmap_get(map, entry);
+    if (found){
+        merge_entry(found, entry);
+    } else {
+        entry_struct *copy = copy_entry(entry);
+        if (copy == NULL) {
+            save_exception(rb_eNoMemError, "Failed copying metrics entry");
+            return 0;
+        }
+        entry_hashmap_put(map, copy, copy); // use the hashmap like hashset actually
+    }
+    return 1;
+}
+
+inline entry_struct entry_new(char* json, size_t json_size, double value, uint32_t pid){
+    return (entry_struct) { .json = json, .json_size = json_size, .pid = pid, .value = value, .name_len = 0 };
+}
+
+void add_parsed_name(entry_struct *entry){
+    jsmn_parser parser;
+    jsmn_init(&parser);
+
+    jsmntok_t t[2];
+    int r = jsmn_parse(&parser, entry->json, entry->json_size, t, 2);
+
+    if (r < 1) {
+        //TODO: better handle this case
+    }
+
+    entry->name = entry->json + t[1].start;
+    entry->name_len = t[1].end - t[1].start;
+}
+
+int entry_lexical_comparator(const entry_struct **a, const entry_struct **b){
+    size_t min_length = min((*a)->json_size, (*b)->json_size);
+    return strncmp((*a)->json, (*b)->json, min_length);
+}
+
+// public functions
+
+void hashmap_setup(struct hashmap *map){
+  hashmap_init(map, (size_t (*)(const void *))hashmap_hash_entry, (int (*)(const void *, const void *))hashmap_compare_entry, 1000);
+}
+
+void entries_destroy(struct hashmap *map){
+    struct hashmap_iter *iter;
+    for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
+        entry_struct *entry = (entry_struct *)entry_hashmap_iter_get_key(iter);
+        entry_free(entry);
+    }
+}
+
+int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map){
+    if (source->size < START_POSITION) {
+        // nothing to read
+        return 1;
+    }
+    uint32_t used;
+    memcpy(&used, source->buffer, sizeof(uint32_t));
+
+    if (used > source->size){
+        //TODO: add custom parsing error
+        save_exception(rb_eRuntimeError, "source file %s corrupted, used %u > file size %u",
+            file_info->path, used, source->size);
+        return 0;
+    }
+
+    uint32_t pos = START_POSITION;
+    while (pos + sizeof(uint32_t) < used) {
+        uint32_t encoded_len;
+        memcpy(&encoded_len, source->buffer + pos, sizeof(uint32_t));
+        pos += sizeof(uint32_t);
+
+        uint32_t value_offset = encoded_len + padding_length(encoded_len);
+
+        if (pos + value_offset + sizeof(double) > used) {
+            //TODO: add custom parsing error
+            save_exception(rb_eRuntimeError, "source file %s corrupted, used %u < stored data length %u",
+                        file_info->path, used, pos + value_offset + sizeof(double));
+            return 0;
+        }
+
+        entry_struct entry = entry_new(source->buffer + pos, encoded_len, 0, 0.0);
+
+        entry.multiprocess_mode = file_info->multiprocess_mode;
+        entry.pid = file_info->pid;
+        entry.type = file_info->type;
+
+        *(entry.json + encoded_len) = '\0'; // TODO: NONO
+
+        pos += value_offset;
+        memcpy(&entry.value, source->buffer + pos, sizeof(double)); //too
+
+        if (!process_entry(map, &entry)) {
+            entries_destroy(map);
+            return 0;
+        }
+
+        pos += sizeof(double);
+    }
+    return 1;
+}
+
+int sort_map_entries(const struct hashmap *map, entry_struct ***sorted_entries){
+    size_t num = hashmap_size(map);
+
+    size_t list_size = num * sizeof(entry_struct *);
+    entry_struct **list = malloc(list_size);
+
+    if (list == NULL){
+        save_exception(rb_eNoMemError, "Couldn't allocate %zu memory", list_size);
+        return 0;
+    }
+
+    size_t cnt = 0;
+    struct hashmap_iter *iter;
+    for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
+        entry_struct *entry = (entry_struct *)entry_hashmap_iter_get_key(iter);
+        add_parsed_name(entry);
+
+        list[cnt] = entry;
+        cnt++;
+    }
+    if (cnt != num){
+        save_exception(rb_eRuntimeError, "Processed entries %zu != map entries %zu", cnt, num);
+        free(list);
+        return 0;
+    }
+
+    qsort(list, cnt, sizeof(entry_struct *), (compare_fn)&entry_lexical_comparator);
+    *sorted_entries = list;
+    return 1;
+}
+