prometheus-client-mmap 0.7.0.beta30 → 0.7.0.beta31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: baae77f6d273813515be523af6a16e159006f636
-  data.tar.gz: 60a044917b7edb908639a22df60727d85136297f
+  metadata.gz: e30cce5a536c75e237464f432aaa103efd53f9af
+  data.tar.gz: c9d27972e59162b19c9b4a71a7d35ea4c1fc849b
 SHA512:
-  metadata.gz: 305cf2a73f788de59de593483847ecb105c8a79b6a98a9f308d1d02a3ac6263b141ae17a546756642cb8ab1e7f9dcaa94b6ae3a9edeeaffcd86628a9ffaf3106
-  data.tar.gz: d3df54b93d82293d7d4a66ca59811296e11b6543128fe7eb354248dd56fe977bfe071d11feb192dcc1247fc20e9dbbfbf2aa255602ff8afbf54233b6c4082684
+  metadata.gz: ce6c424d737eb6b5863008e5a08be2a4172ea1df4fad9eae3246746b90503b849079b77025bc2401895686580c63b7b17e64b760f51189b12b8f0b054c45c79b
+  data.tar.gz: bc6977b2f00f9b0a047cdf0c38c72018cde0e1682e07070d356ee4e7385e710881cd243be6978c8738cb69851635e7bdfb97b137691fdfdf2ea81ec1a500f0a6

data/ext/fast_mmaped_file/extconf.rb

@@ -1,5 +1,7 @@
 require 'mkmf'
-$CFLAGS << ' -Wall -Wextra -Werror'
+$CFLAGS << ' -std=c11 -Wall -Wextra -Werror'
+
+CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
 
 extension_name = 'fast_mmaped_file'
 

data/ext/fast_mmaped_file/fast_mmaped_file.c

@@ -10,25 +10,7 @@
 VALUE MMAPED_FILE = Qnil;
 
 #define START_POSITION 8
-
-VALUE rb_hash_has_key(VALUE hash, VALUE key);
-
-VALUE method_get_double(VALUE self, VALUE index) {
-    mm_ipc *i_mm;
-    GetMmap(self, i_mm, MM_MODIFY);
-
-    Check_Type(index, T_FIXNUM);
-    size_t idx = NUM2UINT(index);
-
-    if ((i_mm->t->real + sizeof(double)) <= idx) {
-        rb_raise(rb_eIndexError, "index %ld out of string", idx);
-    }
-
-    double tmp;
-
-    memcpy(&tmp, (char *)i_mm->t->addr + idx, sizeof(double));
-    return DBL2NUM(tmp);
-}
+#define INITIAL_SIZE (2 * sizeof(int32_t))
 
 void expand(mm_ipc *i_mm, size_t len){
     int fd;
@@ -64,9 +46,46 @@ void expand(mm_ipc *i_mm, size_t len){
     i_mm->t->real = len;
 }
 
+inline uint32_t padding_length(uint32_t key_length){
+    return 8 - (key_length + 4) % 8; // padding | 8 byte aligned
+}
+
+void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
+    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
+
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    memcpy(pos, &key_length, sizeof(uint32_t));
+    pos += sizeof(uint32_t);
+
+    memmove(pos, StringValuePtr(key), key_length);
+    pos += key_length;
+
+    memset(pos, ' ', padding_length(key_length));
+    pos += padding_length(key_length);
+
+    double val = NUM2DBL(value);
+    memcpy(pos, &val, sizeof(double));
+}
+
+inline uint32_t load_used(mm_ipc *i_mm){
+    uint32_t used=0;
+    memcpy(&used, (char *)i_mm->t->addr, sizeof(uint32_t));
+
+    if (used == 0){
+        used = START_POSITION;
+    }
+    return used;
+}
+
+inline void save_used(mm_ipc *i_mm, uint32_t used){
+    memcpy((char *)i_mm->t->addr, &used, sizeof(uint32_t));
+}
+
 VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
     Check_Type(positions, T_HASH);
     Check_Type(key, T_STRING);
+
     VALUE position = rb_hash_lookup(positions, key);
     if (position != Qnil){
         return position;
@@ -75,56 +94,151 @@ VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
     mm_ipc *i_mm;
     GetMmap(self, i_mm, MM_MODIFY);
 
-    if ((i_mm->t->len) < sizeof(int32_t)) {
-        expand(i_mm, sizeof(int32_t)); // TODO: better size
+    if ((i_mm->t->len) < INITIAL_SIZE) {
+        expand(i_mm, INITIAL_SIZE);
     }
 
-    uint32_t used = 0;
-
-    memcpy(&used, (char *)i_mm->t->addr, sizeof(uint32_t));
+    if (i_mm->t->flag & MM_FROZEN){
+        rb_error_frozen("mmap");
+    }
 
-    if (used == 0){
-        used = START_POSITION;
+    if (RSTRING_LEN(key) > UINT32_MAX){
+        rb_raise(rb_eArgError, "string length gt %u", UINT32_MAX);
     }
 
-    long slen = RSTRING_LEN(key);
-    uint32_t padding_length = 8 - (slen + 4) % 8; // padding | 8 byte aligned
-    //TODO: check slen if its less than uint32 throw exception otherwise
-    uint32_t entry_length = (uint32_t)slen;
-    uint32_t new_used = used + sizeof(uint32_t) + entry_length + padding_length + sizeof(double);
+    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
+    uint32_t value_offset = sizeof(uint32_t) + key_length + padding_length(key_length);
+    uint32_t entry_length = value_offset + sizeof(double);
 
-    if (i_mm->t->len < new_used) {
+    uint32_t used = load_used(i_mm);
+    while (i_mm->t->len < (used + entry_length)) {
         expand(i_mm, i_mm->t->len * 2);
     }
 
-    //TODO: add checks if mmap can be written to
+    save_entry(i_mm, used, key, value);
+    save_used(i_mm, used + entry_length);
+    return rb_hash_aset(positions, key, INT2NUM(used + value_offset));
+}
 
-    char *pos = (char *)i_mm->t->addr + used;
+VALUE method_get_double(VALUE self, VALUE index) {
+    mm_ipc *i_mm;
+    GetMmap(self, i_mm, MM_MODIFY);
 
-    memcpy(pos, &entry_length, sizeof(uint32_t));
-    pos += sizeof(uint32_t);
+    Check_Type(index, T_FIXNUM);
+    size_t idx = NUM2UINT(index);
+
+    if ((i_mm->t->real + sizeof(double)) <= idx) {
+        rb_raise(rb_eIndexError, "index %ld out of string", idx);
+    }
 
-    memmove(pos, StringValuePtr(key), slen);
-    pos += slen;
+    double tmp;
 
-    memset(pos, ' ', padding_length);
-    pos += padding_length;
+    memcpy(&tmp, (char *)i_mm->t->addr + idx, sizeof(double));
+    return DBL2NUM(tmp);
+}
 
-    double val = NUM2DBL(value);
-    memcpy(pos, &val, sizeof(double));
+#include <jsmn.h>
+
+#define min(a,b) \
+    ({ __typeof__ (a) _a = (a); \
+       __typeof__ (b) _b = (b); \
+     _a < _b ? _a : _b; })
+
+VALUE token_to_s(char* json, jsmntok_t *t){
+    int len = t->end - t->start;
+    if (len == 0) {
+        return rb_str_new_literal("");
+    } else {
+        return rb_str_new(json + t->start, t->end - t->start);
+    }
+}
+
+VALUE fast_json_parse(char* json, size_t json_size){
+    jsmn_parser parser;
+    jsmn_init(&parser);
 
+    jsmntok_t t[128];
+    int r = jsmn_parse(&parser, json, json_size, t, sizeof(t)/sizeof(t[0]));
 
-    position = INT2NUM(new_used - 8);
-    rb_hash_aset(positions, key, position);
+    if (r < 3) {
+        return Qnil;
+    }
+
+    VALUE labels = rb_hash_new();
+    int label_cnt = (r-5)/2;
+    //TODO: (r - 4) % 2 == 0
+
+    for(int i = 0; i < label_cnt; i++){
+        int key = 4 + i;
+        int val = 5 + label_cnt + i;
+        //TODO: add type checks
 
-    memcpy((char *)i_mm->t->addr, &new_used, sizeof(uint32_t));
+        if (strncmp("null", json + t[val].start, min(4, t[val].end - t[val].start)) == 0) {
+            rb_hash_aset(labels, token_to_s(json, &t[key]), Qnil);
+        } else {
+            rb_hash_aset(labels, token_to_s(json, &t[key]), token_to_s(json, &t[val]));
+        }
+    }
 
-    return position;
+    return rb_ary_new_from_args(3, token_to_s(json, &t[1]), token_to_s(json, &t[2]), labels);
 }
 
 
+VALUE method_fast_json_parse(VALUE self, VALUE source){
+    Check_Type(source, T_STRING);
+    NIL_P(self);
+
+    return fast_json_parse(StringValuePtr(source), RSTRING_LEN(source));
+}
+
+VALUE method_fast_entries(VALUE self, VALUE _data){
+    Check_Type(_data, T_STRING);
+    NIL_P(self);
+
+    char *data = StringValuePtr(_data);
+    uint64_t data_len = RSTRING_LEN(_data);
+
+    uint32_t used;
+    memcpy(&used, data, sizeof(uint32_t));
+
+    uint32_t pos = START_POSITION;
+
+    while (pos < used && (pos + 2*sizeof(uint32_t)) < data_len) {
+        uint32_t encoded_len, first_bytes;
+        memcpy(&encoded_len, data + pos, sizeof(uint32_t));
+        pos += sizeof(uint32_t);
+
+        memcpy(&first_bytes, data + pos, sizeof(uint32_t));
+        if (encoded_len == 0 || first_bytes == 0){
+            pos += 8;
+            continue; // do not parse empty data
+        }
+
+        uint32_t value_offset = encoded_len + padding_length(encoded_len);
+        VALUE metric_name_labels = fast_json_parse(data + pos, encoded_len);
+
+        double value;
+        pos += value_offset;
+        memcpy(&value, data + pos, sizeof(double));
+
+        rb_yield_values(4, rb_ary_entry(metric_name_labels, 0),
+         rb_ary_entry(metric_name_labels, 1),
+         rb_ary_entry(metric_name_labels, 2),
+         DBL2NUM(value)
+         );
+
+        pos += sizeof(double);
+    }
+
+    return Qnil;
+}
+
 void Init_fast_mmaped_file() {
+
     MMAPED_FILE = rb_define_module("FastMmapedFile");
+    rb_define_singleton_method(MMAPED_FILE, "fast_json_parse", method_fast_json_parse, 1);
+    rb_define_singleton_method(MMAPED_FILE, "fast_entries", method_fast_entries, 1);
+
     rb_define_method(MMAPED_FILE, "get_double", method_get_double, 1);
     rb_define_method(MMAPED_FILE, "add_entry", method_add_entry, 3);
 }

data/ext/fast_mmaped_file/jsmn.c

@@ -0,0 +1,313 @@
+#include "jsmn.h"
+
+/**
+ * Allocates a fresh unused token from the token pull.
+ */
+static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser,
+		jsmntok_t *tokens, size_t num_tokens) {
+	jsmntok_t *tok;
+	if (parser->toknext >= num_tokens) {
+		return NULL;
+	}
+	tok = &tokens[parser->toknext++];
+	tok->start = tok->end = -1;
+	tok->size = 0;
+#ifdef JSMN_PARENT_LINKS
+	tok->parent = -1;
+#endif
+	return tok;
+}
+
+/**
+ * Fills token type and boundaries.
+ */
+static void jsmn_fill_token(jsmntok_t *token, jsmntype_t type,
+                            int start, int end) {
+	token->type = type;
+	token->start = start;
+	token->end = end;
+	token->size = 0;
+}
+
+/**
+ * Fills next available token with JSON primitive.
+ */
+static int jsmn_parse_primitive(jsmn_parser *parser, const char *js,
+		size_t len, jsmntok_t *tokens, size_t num_tokens) {
+	jsmntok_t *token;
+	int start;
+
+	start = parser->pos;
+
+	for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+		switch (js[parser->pos]) {
+#ifndef JSMN_STRICT
+			/* In strict mode primitive must be followed by "," or "}" or "]" */
+			case ':':
+#endif
+			case '\t' : case '\r' : case '\n' : case ' ' :
+			case ','  : case ']'  : case '}' :
+				goto found;
+		}
+		if (js[parser->pos] < 32 || js[parser->pos] >= 127) {
+			parser->pos = start;
+			return JSMN_ERROR_INVAL;
+		}
+	}
+#ifdef JSMN_STRICT
+	/* In strict mode primitive must be followed by a comma/object/array */
+	parser->pos = start;
+	return JSMN_ERROR_PART;
+#endif
+
+found:
+	if (tokens == NULL) {
+		parser->pos--;
+		return 0;
+	}
+	token = jsmn_alloc_token(parser, tokens, num_tokens);
+	if (token == NULL) {
+		parser->pos = start;
+		return JSMN_ERROR_NOMEM;
+	}
+	jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos);
+#ifdef JSMN_PARENT_LINKS
+	token->parent = parser->toksuper;
+#endif
+	parser->pos--;
+	return 0;
+}
+
+/**
+ * Fills next token with JSON string.
+ */
+static int jsmn_parse_string(jsmn_parser *parser, const char *js,
+		size_t len, jsmntok_t *tokens, size_t num_tokens) {
+	jsmntok_t *token;
+
+	int start = parser->pos;
+
+	parser->pos++;
+
+	/* Skip starting quote */
+	for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+		char c = js[parser->pos];
+
+		/* Quote: end of string */
+		if (c == '\"') {
+			if (tokens == NULL) {
+				return 0;
+			}
+			token = jsmn_alloc_token(parser, tokens, num_tokens);
+			if (token == NULL) {
+				parser->pos = start;
+				return JSMN_ERROR_NOMEM;
+			}
+			jsmn_fill_token(token, JSMN_STRING, start+1, parser->pos);
+#ifdef JSMN_PARENT_LINKS
+			token->parent = parser->toksuper;
+#endif
+			return 0;
+		}
+
+		/* Backslash: Quoted symbol expected */
+		if (c == '\\' && parser->pos + 1 < len) {
+			int i;
+			parser->pos++;
+			switch (js[parser->pos]) {
+				/* Allowed escaped symbols */
+				case '\"': case '/' : case '\\' : case 'b' :
+				case 'f' : case 'r' : case 'n'  : case 't' :
+					break;
+				/* Allows escaped symbol \uXXXX */
+				case 'u':
+					parser->pos++;
+					for(i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; i++) {
+						/* If it isn't a hex character we have an error */
+						if(!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */
+									(js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */
+									(js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */
+							parser->pos = start;
+							return JSMN_ERROR_INVAL;
+						}
+						parser->pos++;
+					}
+					parser->pos--;
+					break;
+				/* Unexpected symbol */
+				default:
+					parser->pos = start;
+					return JSMN_ERROR_INVAL;
+			}
+		}
+	}
+	parser->pos = start;
+	return JSMN_ERROR_PART;
+}
+
+/**
+ * Parse JSON string and fill tokens.
+ */
+int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
+		jsmntok_t *tokens, unsigned int num_tokens) {
+	int r;
+	int i;
+	jsmntok_t *token;
+	int count = parser->toknext;
+
+	for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+		char c;
+		jsmntype_t type;
+
+		c = js[parser->pos];
+		switch (c) {
+			case '{': case '[':
+				count++;
+				if (tokens == NULL) {
+					break;
+				}
+				token = jsmn_alloc_token(parser, tokens, num_tokens);
+				if (token == NULL)
+					return JSMN_ERROR_NOMEM;
+				if (parser->toksuper != -1) {
+					tokens[parser->toksuper].size++;
+#ifdef JSMN_PARENT_LINKS
+					token->parent = parser->toksuper;
+#endif
+				}
+				token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY);
+				token->start = parser->pos;
+				parser->toksuper = parser->toknext - 1;
+				break;
+			case '}': case ']':
+				if (tokens == NULL)
+					break;
+				type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY);
+#ifdef JSMN_PARENT_LINKS
+				if (parser->toknext < 1) {
+					return JSMN_ERROR_INVAL;
+				}
+				token = &tokens[parser->toknext - 1];
+				for (;;) {
+					if (token->start != -1 && token->end == -1) {
+						if (token->type != type) {
+							return JSMN_ERROR_INVAL;
+						}
+						token->end = parser->pos + 1;
+						parser->toksuper = token->parent;
+						break;
+					}
+					if (token->parent == -1) {
+						if(token->type != type || parser->toksuper == -1) {
+							return JSMN_ERROR_INVAL;
+						}
+						break;
+					}
+					token = &tokens[token->parent];
+				}
+#else
+				for (i = parser->toknext - 1; i >= 0; i--) {
+					token = &tokens[i];
+					if (token->start != -1 && token->end == -1) {
+						if (token->type != type) {
+							return JSMN_ERROR_INVAL;
+						}
+						parser->toksuper = -1;
+						token->end = parser->pos + 1;
+						break;
+					}
+				}
+				/* Error if unmatched closing bracket */
+				if (i == -1) return JSMN_ERROR_INVAL;
+				for (; i >= 0; i--) {
+					token = &tokens[i];
+					if (token->start != -1 && token->end == -1) {
+						parser->toksuper = i;
+						break;
+					}
+				}
+#endif
+				break;
+			case '\"':
+				r = jsmn_parse_string(parser, js, len, tokens, num_tokens);
+				if (r < 0) return r;
+				count++;
+				if (parser->toksuper != -1 && tokens != NULL)
+					tokens[parser->toksuper].size++;
+				break;
+			case '\t' : case '\r' : case '\n' : case ' ':
+				break;
+			case ':':
+				parser->toksuper = parser->toknext - 1;
+				break;
+			case ',':
+				if (tokens != NULL && parser->toksuper != -1 &&
+						tokens[parser->toksuper].type != JSMN_ARRAY &&
+						tokens[parser->toksuper].type != JSMN_OBJECT) {
+#ifdef JSMN_PARENT_LINKS
+					parser->toksuper = tokens[parser->toksuper].parent;
+#else
+					for (i = parser->toknext - 1; i >= 0; i--) {
+						if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) {
+							if (tokens[i].start != -1 && tokens[i].end == -1) {
+								parser->toksuper = i;
+								break;
+							}
+						}
+					}
+#endif
+				}
+				break;
+#ifdef JSMN_STRICT
+			/* In strict mode primitives are: numbers and booleans */
+			case '-': case '0': case '1' : case '2': case '3' : case '4':
+			case '5': case '6': case '7' : case '8': case '9':
+			case 't': case 'f': case 'n' :
+				/* And they must not be keys of the object */
+				if (tokens != NULL && parser->toksuper != -1) {
+					jsmntok_t *t = &tokens[parser->toksuper];
+					if (t->type == JSMN_OBJECT ||
+							(t->type == JSMN_STRING && t->size != 0)) {
+						return JSMN_ERROR_INVAL;
+					}
+				}
+#else
+			/* In non-strict mode every unquoted value is a primitive */
+			default:
+#endif
+				r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens);
+				if (r < 0) return r;
+				count++;
+				if (parser->toksuper != -1 && tokens != NULL)
+					tokens[parser->toksuper].size++;
+				break;
+
+#ifdef JSMN_STRICT
+			/* Unexpected char in strict mode */
+			default:
+				return JSMN_ERROR_INVAL;
+#endif
+		}
+	}
+
+	if (tokens != NULL) {
+		for (i = parser->toknext - 1; i >= 0; i--) {
+			/* Unmatched opened object or array */
+			if (tokens[i].start != -1 && tokens[i].end == -1) {
+				return JSMN_ERROR_PART;
+			}
+		}
+	}
+
+	return count;
+}
+
+/**
+ * Creates a new parser based over a given  buffer with an array of tokens
+ * available.
+ */
+void jsmn_init(jsmn_parser *parser) {
+	parser->pos = 0;
+	parser->toknext = 0;
+	parser->toksuper = -1;
+}

data/ext/fast_mmaped_file/jsmn.h

@@ -0,0 +1,76 @@
+#ifndef __JSMN_H_
+#define __JSMN_H_
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * JSON type identifier. Basic types are:
+ * 	o Object
+ * 	o Array
+ * 	o String
+ * 	o Other primitive: number, boolean (true/false) or null
+ */
+typedef enum {
+	JSMN_UNDEFINED = 0,
+	JSMN_OBJECT = 1,
+	JSMN_ARRAY = 2,
+	JSMN_STRING = 3,
+	JSMN_PRIMITIVE = 4
+} jsmntype_t;
+
+enum jsmnerr {
+	/* Not enough tokens were provided */
+	JSMN_ERROR_NOMEM = -1,
+	/* Invalid character inside JSON string */
+	JSMN_ERROR_INVAL = -2,
+	/* The string is not a full JSON packet, more bytes expected */
+	JSMN_ERROR_PART = -3
+};
+
+/**
+ * JSON token description.
+ * type		type (object, array, string etc.)
+ * start	start position in JSON data string
+ * end		end position in JSON data string
+ */
+typedef struct {
+	jsmntype_t type;
+	int start;
+	int end;
+	int size;
+#ifdef JSMN_PARENT_LINKS
+	int parent;
+#endif
+} jsmntok_t;
+
+/**
+ * JSON parser. Contains an array of token blocks available. Also stores
+ * the string being parsed now and current position in that string
+ */
+typedef struct {
+	unsigned int pos; /* offset in the JSON string */
+	unsigned int toknext; /* next token to allocate */
+	int toksuper; /* superior token node, e.g parent object or array */
+} jsmn_parser;
+
+/**
+ * Create JSON parser over an array of tokens
+ */
+void jsmn_init(jsmn_parser *parser);
+
+/**
+ * Run JSON parser. It parses a JSON data string into and array of tokens, each describing
+ * a single JSON object.
+ */
+int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
+		jsmntok_t *tokens, unsigned int num_tokens);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __JSMN_H_ */

data/lib/prometheus/client/formats/text.rb

@@ -1,5 +1,8 @@
 require 'prometheus/client/uses_value_type'
 require 'prometheus/client/helper/json_parser'
+require 'prometheus/client/helper/plain_file'
+require 'prometheus/client/helper/metrics_processing'
+require 'prometheus/client/helper/metrics_representation'
 
 module Prometheus
   module Client
@@ -10,166 +13,68 @@ module Prometheus
         VERSION = '0.0.4'.freeze
         CONTENT_TYPE = "#{MEDIA_TYPE}; version=#{VERSION}".freeze
 
-        METRIC_LINE = '%s%s %s'.freeze
-        TYPE_LINE = '# TYPE %s %s'.freeze
-        HELP_LINE = '# HELP %s %s'.freeze
-
-        LABEL = '%s="%s"'.freeze
-        SEPARATOR = ','.freeze
-        DELIMITER = "\n".freeze
-
-        REGEX = { doc: /[\n\\]/, label: /[\n\\"]/ }.freeze
-        REPLACE = { "\n" => '\n', '\\' => '\\\\', '"' => '\"' }.freeze
-
         class << self
           def marshal(registry)
-            lines = []
 
-            registry.metrics.each do |metric|
-              lines << format(HELP_LINE, metric.name, escape(metric.docstring))
-              lines << format(TYPE_LINE, metric.name, metric.type)
-              metric.values.each do |label_set, value|
-                representation(metric, label_set, value) { |l| lines << l }
+            metrics = registry.metrics.map do |metric|
+              samples = metric.values.flat_map do |label_set, value|
+                representation(metric, label_set, value)
               end
+
+              [metric.name, { type: metric.type, help: metric.docstring, samples: samples }]
             end
 
-            # there must be a trailing delimiter
-            (lines << nil).join(DELIMITER)
+            Helper::MetricsRepresentation.to_text(metrics)
           end
 
           def marshal_multiprocess(path = Prometheus::Client.configuration.multiprocess_files_dir)
             metrics = load_metrics(path)
 
-            lines = []
-            merge_metrics(metrics).each do |name, metric|
-              lines << format(HELP_LINE, name, escape(metric[:help]))
-              lines << format(TYPE_LINE, name, metric[:type])
-
-              metric[:samples].each do |metric_name, labels, value|
-                lines << metric(metric_name, format_labels(labels), value)
-              end
-            end
-            (lines << nil).join(DELIMITER)
+            Helper::MetricsRepresentation.to_text(Helper::MetricsProcessing.merge_metrics(metrics))
           end
 
           private
 
-          def merge_metrics(metrics)
-            metrics.each_value do |metric|
-              metric[:samples] = merge_samples(metric[:samples], metric[:type], metric[:multiprocess_mode]).map do |(name, labels), value|
-                [name, labels.to_h, value]
-              end
-            end
-          end
-
-          def merge_samples(raw_samples, metric_type, multiprocess_mode)
-            samples = {}
-            raw_samples.each do |name, labels, value|
-              without_pid = labels.reject { |l| l[0] == 'pid' }
-
-              case metric_type
-                when :gauge
-                  case multiprocess_mode
-                    when 'min'
-                      s = samples.fetch([name, without_pid], value)
-                      samples[[name, without_pid]] = [s, value].min
-                    when 'max'
-                      s = samples.fetch([name, without_pid], value)
-                      samples[[name, without_pid]] = [s, value].max
-                    when 'livesum'
-                      s = samples.fetch([name, without_pid], 0.0)
-                      samples[[name, without_pid]] = s + value
-                    else # all/liveall
-                      samples[[name, labels]] = value
-                  end
-                else
-                  # Counter, Histogram and Summary.
-                  s = samples.fetch([name, without_pid], 0.0)
-                  samples[[name, without_pid]] = s + value
-              end
-            end
-
-            samples
-          end
-
           def load_metrics(path)
             metrics = {}
             Dir.glob(File.join(path, '*.db')).sort.each do |f|
-              parts = File.basename(f, '.db').split('_')
-              parts[-1].gsub!(/(?<=.)-\d+/, '') # remove trailing file number
-              type = parts[0].to_sym
-
-              MmapedDict.read_all_values(f).each do |key, value|
-                metric_name, name, labelnames, labelvalues = Helper::JsonParser.load(key)
-                metric = metrics.fetch(metric_name,
-                                       metric_name: metric_name,
-                                       help: 'Multiprocess metric',
-                                       type: type,
-                                       samples: []
-                )
-                if type == :gauge
-                  pid = parts[2..-1].join('_')
-                  metric[:multiprocess_mode] = parts[1]
-                  metric[:samples] += [[name, labelnames.zip(labelvalues) + [['pid', pid]], value]]
-                else
-                  # The duplicates and labels are fixed in the next for.
-                  metric[:samples] += [[name, labelnames.zip(labelvalues), value]]
-                end
-                metrics[metric_name] = metric
-              end
+              Helper::PlainFile.new(f).to_metrics(metrics)
             end
             metrics
           end
 
-          def representation(metric, label_set, value, &block)
-            set = metric.base_labels.merge(label_set)
+          def representation(metric, label_set, value)
+            labels = metric.base_labels.merge(label_set)
 
             if metric.type == :summary
-              summary(metric.name, set, value, &block)
+              summary(metric.name, labels, value)
             elsif metric.type == :histogram
-              histogram(metric.name, set, value, &block)
+              histogram(metric.name, labels, value)
             else
-              yield metric(metric.name, format_labels(set), value.get)
+              [[metric.name, labels, value.get]]
             end
           end
 
           def summary(name, set, value)
-            value.get.each do |q, v|
-              yield metric(name, format_labels(set.merge(quantile: q)), v)
+            rv = value.get.map do |q, v|
+              [name, set.merge(quantile: q), v]
             end
 
-            l = format_labels(set)
-            yield metric("#{name}_sum", l, value.get.sum)
-            yield metric("#{name}_count", l, value.get.total)
+            rv << ["#{name}_sum", set, value.get.sum]
+            rv << ["#{name}_count", set, value.get.total]
+            rv
           end
 
           def histogram(name, set, value)
-            value.get.each do |q, v|
-              yield metric(name, format_labels(set.merge(le: q)), v)
-            end
-            yield metric(name, format_labels(set.merge(le: '+Inf')), value.get.total)
-
-            l = format_labels(set)
-            yield metric("#{name}_sum", l, value.get.sum)
-            yield metric("#{name}_count", l, value.get.total)
-          end
-
-          def metric(name, labels, value)
-            format(METRIC_LINE, name, labels, value)
-          end
-
-          def format_labels(set)
-            return if set.empty?
-
-            strings = set.each_with_object([]) do |(key, value), memo|
-              memo << format(LABEL, key, escape(value, :label))
+            # |metric_name, labels, value|
+            rv = value.get.map do |q, v|
+              [name, set.merge(le: q), v]
             end
 
-            "{#{strings.join(SEPARATOR)}}"
-          end
-
-          def escape(string, format = :doc)
-            string.to_s.gsub(REGEX[format], REPLACE)
+            rv << [name, set.merge(le: '+Inf'), value.get.total]
+            rv << ["#{name}_sum", set, value.get.sum]
+            rv << ["#{name}_count", set, value.get.total]
+            rv
           end
         end
       end

data/lib/prometheus/client/helper/entry_parser.rb

@@ -1,7 +1,12 @@
+require 'prometheus/client/helper/json_parser'
+require 'fast_mmaped_file'
+
 module Prometheus
   module Client
     module Helper
       module EntryParser
+        class ParsingError < RuntimeError;
+        end
         MINIMUM_SIZE = 8
         START_POSITION = 8
 
@@ -9,6 +14,24 @@ module Prometheus
           slice(0..3).unpack('l')[0]
         end
 
+        def parts
+          @parts ||= File.basename(filepath, '.db')
+                       .split('_')
+                       .map { |e| e.gsub(/-\d+$/, '') } # remove trailing -number
+        end
+
+        def type
+          parts[0].to_sym
+        end
+
+        def pid
+          parts[2..-1].join('_')
+        end
+
+        def multiprocess_mode
+          parts[1]
+        end
+
         def empty?
           size < MINIMUM_SIZE || used.zero?
         end
@@ -38,6 +61,101 @@ module Prometheus
             end
           end
         end
+
+        def parsed_entries(ignore_errors = false)
+          result = entries.map do |data, encoded_len, value_offset, _|
+            begin
+              encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset))
+              [encoded, value]
+            rescue ArgumentError => e
+              Prometheus::Client.logger.debug("Error processing data: #{bin_to_hex(data[0, 7])} len: #{encoded_len} value_offset: #{value_offset}")
+              raise ParsingError(e) unless ignore_errors
+            end
+          end
+
+          result.reject { |e| e.nil? } if ignore_errors
+          result
+        end
+
+        def to_metrics(metrics = {}, ignore_errors = false)
+          FastMmapedFile.fast_entries(data) do |metric_name, name, labels, value|
+            begin
+              metric = metrics.fetch(metric_name,
+                                     metric_name: metric_name,
+                                     help: 'Multiprocess metric',
+                                     type: type,
+                                     samples: {}
+              )
+              if type == :gauge
+                metric[:multiprocess_mode] = multiprocess_mode
+                merge_samples(name, labels, value, metric[:samples])
+              else
+                merge_samples(name, labels, value, metric[:samples])
+                # The duplicates and labels are fixed in the next for.
+              end
+              metrics[metric_name] = metric
+            rescue JSON::ParserError => e
+              raise ParsingError(e) unless ignore_errors
+            end
+          end
+
+          metrics.reject { |e| e.nil? } if ignore_errors
+          metrics
+        end
+
+
+        def min(a, b)
+          if a < b
+            a
+          else
+            b
+          end
+        end
+
+        def max(a, b)
+          if a > b
+            a
+          else
+            b
+          end
+        end
+
+        def merge_samples(name, labels, value, samples)
+          samples ||= {}
+          case type
+            when :gauge
+              case multiprocess_mode
+                when 'min'
+                  key = labels.merge!(__key_name: name)
+                  s = samples.fetch(key, value)
+                  samples[key] = min(s, value)
+                when 'max'
+                  key = labels.merge!(__key_name: name)
+                  s = samples.fetch(key, value)
+                  samples[key] = max(s, value)
+                when 'livesum'
+                  key = labels.merge!(__key_name: name)
+                  s = samples.fetch(key, 0.0)
+                  samples[key] = s + value
+                else # all/liveall
+                  key = labels.merge!(pid: pid, __key_name: name)
+                  samples[key] = value
+              end
+            else
+              key = labels.merge!(__key_name: name)
+              # Counter, Histogram and Summary.
+              s = samples.fetch(key, 0.0)
+              samples[key] = s + value
+          end
+
+          samples
+        end
+
+        private
+
+        def bin_to_hex(s)
+          s.each_byte.map { |b| b.to_s(16) }.join
+        end
       end
     end
   end

data/lib/prometheus/client/helper/json_parser.rb

@@ -1,3 +1,5 @@
+require 'json'
+
 module Prometheus
   module Client
     module Helper

data/lib/prometheus/client/helper/metrics_processing.rb

@@ -0,0 +1,62 @@
+module Prometheus
+  module Client
+    module Helper
+      module MetricsProcessing
+        def self.merge_metrics(metrics)
+          metrics
+        end
+
+        def self.min(a, b)
+          if a < b
+            a
+          else
+            b
+          end
+        end
+
+        def self.max(a, b)
+          if a > b
+            a
+          else
+            b
+          end
+        end
+
+        def self.merge_samples(raw_samples, metric_type, multiprocess_mode)
+          samples = {}
+          raw_samples.each do |name, labels, value|
+            key = { name: name, labels: labels }
+            # without_pid = labels.delete(:pid)
+
+            case metric_type
+              when :gauge
+                case multiprocess_mode
+                  when 'min'
+                    labels.delete(:pid)
+                    s = samples.fetch(key, value)
+                    samples[key] = min(s, value)
+                  when 'max'
+                    labels.delete(:pid)
+                    s = samples.fetch(key, value)
+                    samples[key] = max(s, value)
+                  when 'livesum'
+                    labels.delete(:pid)
+                    s = samples.fetch(key, 0.0)
+                    samples[key] = s + value
+                  else # all/liveall
+                    samples[key] = value
+                end
+              else
+                labels.delete(:pid)
+                # Counter, Histogram and Summary.
+                s = samples.fetch(key, 0.0)
+                samples[key] = s + value
+            end
+          end
+
+          samples
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/metrics_representation.rb

@@ -0,0 +1,52 @@
+module Prometheus
+  module Client
+    module Helper
+      module MetricsRepresentation
+        METRIC_LINE = '%s%s %s'.freeze
+        TYPE_LINE = '# TYPE %s %s'.freeze
+        HELP_LINE = '# HELP %s %s'.freeze
+
+        LABEL = '%s="%s"'.freeze
+        SEPARATOR = ','.freeze
+        DELIMITER = "\n".freeze
+
+        REGEX = { doc: /[\n\\]/, label: /[\n\\"]/ }.freeze
+        REPLACE = { "\n" => '\n', '\\' => '\\\\', '"' => '\"' }.freeze
+
+        def self.to_text(metrics)
+          lines = []
+
+          metrics.each do |name, metric|
+            lines << format(HELP_LINE, name, escape(metric[:help]))
+            lines << format(TYPE_LINE, name, metric[:type])
+            metric[:samples].each do |labels, value|
+              key = labels.delete(:__key_name)
+              lines << metric(key, format_labels(labels), value)
+            end
+          end
+
+          # there must be a trailing delimiter
+          (lines << nil).join(DELIMITER)
+        end
+
+        def self.metric(name, labels, value)
+          format(METRIC_LINE, name, labels, value)
+        end
+
+        def self.format_labels(set) #TODO: rename set
+          return if set.empty?
+
+          strings = set.each_with_object([]) do |(key, value), memo|
+            memo << format(LABEL, key, escape(value, :label))
+          end
+
+          "{#{strings.join(SEPARATOR)}}"
+        end
+
+        def self.escape(string, format = :doc)
+          string.to_s.gsub(REGEX[format], REPLACE)
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/helper/plain_file.rb

@@ -5,8 +5,10 @@ module Prometheus
     module Helper
       class PlainFile
         include EntryParser
+        attr_reader :filepath, :data
 
         def initialize(filepath)
+          @filepath = filepath
           @data = File.read(filepath, mode: 'rb')
         end
 

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '0.7.0.beta30'.freeze
+    VERSION = '0.7.0.beta31'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 0.7.0.beta30
+  version: 0.7.0.beta31
 platform: ruby
 authors:
 - Tobias Schmidt
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-17 00:00:00.000000000 Z
+date: 2017-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mmap2
@@ -71,6 +71,8 @@ files:
 - README.md
 - ext/fast_mmaped_file/extconf.rb
 - ext/fast_mmaped_file/fast_mmaped_file.c
+- ext/fast_mmaped_file/jsmn.c
+- ext/fast_mmaped_file/jsmn.h
 - ext/fast_mmaped_file/mmap.h
 - lib/fast_mmaped_file.bundle
 - lib/prometheus.rb
@@ -82,6 +84,8 @@ files:
 - lib/prometheus/client/helper/entry_parser.rb
 - lib/prometheus/client/helper/file_locker.rb
 - lib/prometheus/client/helper/json_parser.rb
+- lib/prometheus/client/helper/metrics_processing.rb
+- lib/prometheus/client/helper/metrics_representation.rb
 - lib/prometheus/client/helper/mmaped_file.rb
 - lib/prometheus/client/helper/plain_file.rb
 - lib/prometheus/client/histogram.rb
@@ -118,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: A suite of instrumentation metric primitivesthat can be exposed through a