prometheus-client-mmap 0.7.0.beta39 → 0.7.0.beta40

data/ext/fast_mmaped_file/jsmn.h

@@ -0,0 +1,76 @@
+#ifndef __JSMN_H_
+#define __JSMN_H_
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * JSON type identifier. Basic types are:
+ * 	o Object
+ * 	o Array
+ * 	o String
+ * 	o Other primitive: number, boolean (true/false) or null
+ */
+typedef enum {
+	JSMN_UNDEFINED = 0,
+	JSMN_OBJECT = 1,
+	JSMN_ARRAY = 2,
+	JSMN_STRING = 3,
+	JSMN_PRIMITIVE = 4
+} jsmntype_t;
+
+enum jsmnerr {
+	/* Not enough tokens were provided */
+	JSMN_ERROR_NOMEM = -1,
+	/* Invalid character inside JSON string */
+	JSMN_ERROR_INVAL = -2,
+	/* The string is not a full JSON packet, more bytes expected */
+	JSMN_ERROR_PART = -3
+};
+
+/**
+ * JSON token description.
+ * type		type (object, array, string etc.)
+ * start	start position in JSON data string
+ * end		end position in JSON data string
+ */
+typedef struct {
+	jsmntype_t type;
+	int start;
+	int end;
+	int size;
+#ifdef JSMN_PARENT_LINKS
+	int parent;
+#endif
+} jsmntok_t;
+
+/**
+ * JSON parser. Contains an array of token blocks available. Also stores
+ * the string being parsed now and current position in that string
+ */
+typedef struct {
+	unsigned int pos; /* offset in the JSON string */
+	unsigned int toknext; /* next token to allocate */
+	int toksuper; /* superior token node, e.g parent object or array */
+} jsmn_parser;
+
+/**
+ * Create JSON parser over an array of tokens
+ */
+void jsmn_init(jsmn_parser *parser);
+
+/**
+ * Run JSON parser. It parses a JSON data string into and array of tokens, each describing
+ * a single JSON object.
+ */
+int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
+		jsmntok_t *tokens, unsigned int num_tokens);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __JSMN_H_ */

data/ext/fast_mmaped_file/mmap.h

@@ -1,6 +1,8 @@
 #ifndef MMAP_H
 #define MMAP_H
 
+#include <ruby.h>
+
 #define MM_MODIFY 1
 #define MM_ORIGIN 2
 #define MM_CHANGE (MM_MODIFY | 4)

data/ext/fast_mmaped_file/rendering.c

@@ -0,0 +1,92 @@
+#include "rendering.h"
+#include <jsmn.h>
+#include <file_parsing.h>
+#include <utils.h>
+
+int not_null(const entry_struct *entry, const jsmntok_t *t){
+    return strncmp("null", entry->json + t->start, min(4, t->end - t->start)) != 0;
+}
+
+inline void append_token(VALUE string, const entry_struct *entry, const jsmntok_t *t){
+    rb_str_cat(string, entry->json + t->start, t->end - t->start); //TODO: add token sanity checks
+}
+
+void append_entry(VALUE string, const entry_struct *entry){
+    jsmn_parser parser;
+    jsmn_init(&parser);
+
+    jsmntok_t t[128]; //TODO: document how many labels we can process
+    int r = jsmn_parse(&parser, entry->json, entry->json_size, t, sizeof(t)/sizeof(t[0]));
+
+    if (r < 3) {
+        return; //TODO: better handle this case
+    }
+
+    //TODO: check (r - 4) % 2 == 0
+    int label_cnt = (r-5)/2;
+
+    append_token(string, entry, &t[2]);
+    if (label_cnt > 0){
+        rb_str_cat(string, "{", 1);
+
+        for(int i = 0; i < label_cnt; i++){
+            int key = 4 + i;
+            int val = 5 + label_cnt + i;
+            //TODO: add type checks
+            append_token(string, entry, &t[key]);
+            rb_str_cat(string, "=", 1);
+
+            rb_str_cat(string, "\"", 1);
+            if (not_null(entry, &t[val])){
+                append_token(string, entry, &t[val]);
+            }
+            rb_str_cat(string, "\"", 1);
+
+            if (i < label_cnt - 1){
+                rb_str_cat(string, ",", 1);
+            }
+        }
+        rb_str_cat(string, "}", 1);
+    }
+
+    char value[255];
+
+    //TODO: fix value representation %g doesn't math what Go expects
+    int written = snprintf(value, sizeof(value), " %g\n", entry->value);
+    rb_str_cat(string, value, written);
+}
+
+void append_entry_head(VALUE string, const entry_struct *entry){
+    static const char help_beg[] = "# HELP ";
+    static const char help_fin[] = " Multiprocess metric\n";
+
+    rb_str_cat(string, help_beg, sizeof(help_beg) - 1);
+    rb_str_cat(string, entry->name, entry->name_len);
+    rb_str_cat(string, help_fin, sizeof(help_fin) - 1);
+
+    static const char type_beg[] = "# TYPE ";
+
+    rb_str_cat(string, type_beg, sizeof(type_beg) - 1);
+    rb_str_cat(string, entry->name, entry->name_len);
+    rb_str_cat(string, " ", 1);
+    rb_str_cat2(string, rb_id2name(entry->type));
+    rb_str_cat(string, "\n", 1);
+}
+
+VALUE entries_to_string(entry_struct **sorted_entries, size_t entries_count){
+    VALUE rv = rb_str_new("", 0);
+
+    entry_struct *previous = NULL;
+
+    for(size_t i = 0; i < entries_count; i ++){
+        entry_struct *entry = sorted_entries[i];
+        if (!previous || previous->name_len != entry->name_len || strncmp(entry->name, previous->name, entry->name_len) != 0) {
+            previous = entry;
+            append_entry_head(rv, entry);
+        }
+
+        append_entry(rv, entry);
+    }
+
+    return rv;
+}

data/ext/fast_mmaped_file/rendering.h

@@ -0,0 +1,8 @@
+#ifndef RENDERING_H
+#define RENDERING_H
+#include <ruby.h>
+#include <file_parsing.h>
+
+VALUE entries_to_string(entry_struct **sorted_entries, size_t entries_count);
+
+#endif

data/ext/fast_mmaped_file/utils.c

@@ -0,0 +1,25 @@
+#include <ruby.h>
+#include "utils.h"
+
+void save_exception(VALUE exception, const char *fmt, ...){
+    va_list args;
+
+    va_start(args, fmt);
+    VALUE message = rb_vsprintf(fmt, args);
+    va_end(args);
+
+    VALUE current_thread = rb_thread_current();
+    rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception"), exception);
+    rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception_message"), message);
+}
+
+void raise_last_exception(){
+    VALUE current_thread = rb_thread_current();
+    VALUE exception = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception"));
+    VALUE message = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception_message"));
+    if (exception != Qnil) {
+        rb_raise(exception, "%s", StringValueCStr(message));
+    } else {
+        rb_raise(rb_eRuntimeError, "no exception found in thread local");
+    }
+}

data/ext/fast_mmaped_file/utils.h

@@ -0,0 +1,26 @@
+#ifndef UNUSED_H
+#define UNUSED_H
+
+#ifdef UNUSED
+#elif defined(__GNUC__)
+#define UNUSED(x) UNUSED_ ## x __attribute__((unused))
+#elif defined(__LCLINT__)
+#define UNUSED(x) /*@unused@*/ x
+#else
+#define UNUSED(x) x
+#endif
+
+#define min(a,b) \
+    ({ __typeof__ (a) _a = (a); \
+       __typeof__ (b) _b = (b); \
+     _a < _b ? _a : _b; })
+
+#define max(a,b) \
+    ({ __typeof__ (a) _a = (a); \
+       __typeof__ (b) _b = (b); \
+     _a > _b ? _a : _b; })
+
+void raise_last_exception();
+void save_exception(VALUE exception, const char *fmt, ...);
+
+#endif

data/ext/fast_mmaped_file/value_access.c

@@ -0,0 +1,160 @@
+#include <sys/mman.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include "file_format.h"
+#include "mmap.h"
+#include "value_access.h"
+
+int open_and_extend_file(mm_ipc *i_mm, size_t len) {
+    int fd;
+
+    if ((fd = open(i_mm->t->path, i_mm->t->smode)) == -1) {
+        rb_raise(rb_eArgError, "Can't open %s", i_mm->t->path);
+    }
+
+    if (lseek(fd, len - i_mm->t->len - 1, SEEK_END) == -1) {
+        close(fd);
+        rb_raise(rb_eIOError, "Can't lseek %lu", len - i_mm->t->len - 1);
+    }
+
+    if (write(fd, "\000", 1) != 1) {
+        close(fd);
+        rb_raise(rb_eIOError, "Can't extend %s", i_mm->t->path);
+    }
+
+    return fd;
+}
+
+void expand(mm_ipc *i_mm, size_t len) {
+    if (len < i_mm->t->len) {
+        rb_raise(rb_eArgError, "Can't reduce the size of mmap");
+    }
+
+    if (munmap(i_mm->t->addr, i_mm->t->len)) {
+        rb_raise(rb_eArgError, "munmap failed");
+    }
+
+    int fd = open_and_extend_file(i_mm, len);
+
+    i_mm->t->addr = mmap(0, len, i_mm->t->pmode, i_mm->t->vscope, fd, i_mm->t->offset);
+
+    if (i_mm->t->addr == MAP_FAILED) {
+        close(fd);
+        rb_raise(rb_eArgError, "mmap failed");
+    }
+
+    if (close(fd) == -1) {
+        rb_raise(rb_eArgError, "Can't close %s", i_mm->t->path);
+    }
+
+    if ((i_mm->t->flag & MM_LOCK) && mlock(i_mm->t->addr, len) == -1) {
+        rb_raise(rb_eArgError, "mlock(%d)", errno);
+    }
+    i_mm->t->len = len;
+    i_mm->t->real = len;
+}
+
+void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
+    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
+
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    memcpy(pos, &key_length, sizeof(uint32_t));
+    pos += sizeof(uint32_t);
+
+    memmove(pos, StringValuePtr(key), key_length);
+    pos += key_length;
+
+    memset(pos, ' ', padding_length(key_length));
+    pos += padding_length(key_length);
+
+    double val = NUM2DBL(value);
+    memcpy(pos, &val, sizeof(double));
+}
+
+inline uint32_t load_used(mm_ipc *i_mm) {
+    uint32_t used = *((uint32_t *)i_mm->t->addr);
+
+    if (used == 0) {
+        used = START_POSITION;
+    }
+    return used;
+}
+
+inline void save_used(mm_ipc *i_mm, uint32_t used) {
+    *((uint32_t *)i_mm->t->addr) = used;
+}
+
+VALUE method_load_used(VALUE self) {
+    mm_ipc *i_mm;
+
+    GET_MMAP(self, i_mm, MM_MODIFY);
+    return UINT2NUM(load_used(i_mm));
+}
+
+VALUE method_save_used(VALUE self, VALUE value) {
+    Check_Type(value, T_FIXNUM);
+    mm_ipc *i_mm;
+
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    if (i_mm->t->len < INITIAL_SIZE) {
+        expand(i_mm, INITIAL_SIZE);
+    }
+
+    save_used(i_mm, NUM2UINT(value));
+    return value;
+}
+
+VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
+    Check_Type(positions, T_HASH);
+    Check_Type(key, T_STRING);
+
+    VALUE position = rb_hash_lookup(positions, key);
+    if (position != Qnil){
+        return position;
+    }
+
+    mm_ipc *i_mm;
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
+    }
+
+    if (RSTRING_LEN(key) > UINT32_MAX) {
+        rb_raise(rb_eArgError, "string length gt %u", UINT32_MAX);
+    }
+
+    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
+    uint32_t value_offset = sizeof(uint32_t) + key_length + padding_length(key_length);
+    uint32_t entry_length = value_offset + sizeof(double);
+
+    uint32_t used = load_used(i_mm);
+    while (i_mm->t->len < (used + entry_length)) {
+        expand(i_mm, i_mm->t->len * 2);
+    }
+
+    save_entry(i_mm, used, key, value);
+    save_used(i_mm, used + entry_length);
+    return rb_hash_aset(positions, key, INT2NUM(used + value_offset));
+}
+
+VALUE method_get_double(VALUE self, VALUE index) {
+    mm_ipc *i_mm;
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    Check_Type(index, T_FIXNUM);
+    size_t idx = NUM2UINT(index);
+
+    if ((i_mm->t->real + sizeof(double)) <= idx) {
+        rb_raise(rb_eIndexError, "index %ld out of string", idx);
+    }
+
+    double tmp;
+
+    memcpy(&tmp, (char *)i_mm->t->addr + idx, sizeof(double));
+    return DBL2NUM(tmp);
+}

data/ext/fast_mmaped_file/value_access.h

@@ -0,0 +1,14 @@
+#ifndef VALUE_ACCESS_H
+#define VALUE_ACCESS_H
+
+#include <ruby.h>
+
+VALUE method_load_used(VALUE self);
+
+VALUE method_save_used(VALUE self, VALUE value);
+
+VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value);
+
+VALUE method_get_double(VALUE self, VALUE index);
+
+#endif

data/lib/prometheus/client/formats/text.rb

@@ -27,9 +27,11 @@ module Prometheus
           end
 
           def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir)
-            metrics = load_metrics(path)
+            file_list = Dir.glob(File.join(path, '*.db')).sort
+              .map {|f| Helper::PlainFile.new(f) }
+              .map {|f| [f.filepath, f.multiprocess_mode.to_sym, f.type.to_sym, f.pid] }
 
-            Helper::MetricsRepresentation.to_text(Helper::MetricsProcessing.merge_metrics(metrics))
+            FastMmapedFile.to_metrics(file_list.to_a)
           end
 
           private

data/lib/prometheus/client/helper/plain_file.rb

@@ -8,17 +8,20 @@ module Prometheus
         include EntryParser
         attr_reader :filepath
 
+        def data
+          @data ||= File.read(filepath, mode: 'rb')
+        end
+
         def initialize(filepath)
           @filepath = filepath
-          @data = File.read(filepath, mode: 'rb')
         end
 
         def slice(*args)
-          @data.slice(*args)
+          data.slice(*args)
         end
 
         def size
-          @data.length
+          data.length
         end
       end
     end