prometheus-client-mmap 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d920ebe3faf6dc2b32bea11683d225fba6b1fed
-  data.tar.gz: '090323b8d9a5d05db075dc13dc105b66110528ef'
+  metadata.gz: d8752d354419c2916e63d9d02e2891093725d6d0
+  data.tar.gz: 2abff88dd92d618f310ede7213e061b74922c164
 SHA512:
-  metadata.gz: 400c904f7f704eb43184cb05dbe8106bf775d89f82fc0f5de91b707971202e9f809497c623cabc5f0925c38a056d854701769f524c0720414e49a94f6dcfa538
-  data.tar.gz: c6282e0839cf5d18292222a9d0401a09d5569f244dc291e35baf528dcf1b54acf222a7e71045ff054e77d739fb71884af3c0c11bc76df85b1ce7e54a23901b16
+  metadata.gz: 7e1f15bf7e89250e0b0fdff424a4e1fc1e06f35d333b0cb633d2b40bb53bd8d1b7cc94b8181d578369000593fe59319cb7db28f345654d0a5eee40314fd09811
+  data.tar.gz: 9464dd49c6fd0d273ff1790c2524798dd54b5c2adc4ed911bd5901e2c5a7ce213e78a13d7dd024cac6a1afef3af46151c8903281d77112ec42877a58d36c01c5

data/ext/fast_mmaped_file/#file_parsing.c#

@@ -0,0 +1,216 @@
+#include <hashmap.h>
+#include <jsmn.h>
+#include <ruby.h>
+
+#include "file_format.h"
+#include "file_parsing.h"
+#include "globals.h"
+#include "utils.h"
+
+HASHMAP_FUNCS_CREATE(entry, const entry_t, entry_t)
+
+typedef int (*compare_fn)(const void *a, const void *b);
+
+static size_t hashmap_hash_entry(const entry_t *entry) {
+    size_t hash = 0;
+
+    for (size_t i = 0; i < entry->json_size; i++) {
+        hash += *(const char *)(entry->json + i);
+        hash += (hash << 10);
+        hash ^= (hash >> 6);
+    }
+    hash += (hash << 3);
+    hash ^= (hash >> 11);
+    hash += (hash << 15);
+    return hash;
+}
+
+static int hashmap_compare_entry(const entry_t *a, const entry_t *b) {
+    if (a->json_size != b->json_size) {
+        return -1;
+    }
+
+    if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)) {
+        return -1;
+    }
+
+    return strncmp(a->json, b->json, a->json_size);
+}
+
+static entry_t *copy_entry(const entry_t *entry) {
+    entry_t *copied = (entry_t *)malloc(sizeof(entry_t));
+    if (copied == NULL) {
+        return NULL;
+    }
+    memcpy(copied, entry, sizeof(entry_t));
+
+    copied->json = malloc(entry->json_size);
+    if (copied->json == NULL) {
+        free(copied);
+        return NULL;
+    }
+
+    memcpy(copied->json, entry->json, entry->json_size);
+
+    return copied;
+}
+
+static void entry_free(entry_t *entry) {
+    free(entry->json);
+    free(entry);
+}
+
+static void merge_entry(entry_t *found, const entry_t *entry) {
+    if (entry->type == sym_gauge) {
+        if (entry->multiprocess_mode == sym_min) {
+            found->value = min(found->value, entry->value);
+        } else if (entry->multiprocess_mode == sym_max) {
+            found->value = max(found->value, entry->value);
+        } else if (entry->multiprocess_mode == sym_livesum) {
+            found->value += entry->value;
+        } else {
+            found->value = entry->value;
+        }
+    } else {
+        found->value += entry->value;
+    }
+}
+
+static int process_entry(struct hashmap *map, const entry_t *entry) {
+    entry_t *found = entry_hashmap_get(map, entry);
+    if (found) {
+        merge_entry(found, entry);
+    } else {
+        entry_t *copy = copy_entry(entry);
+        if (copy == NULL) {
+            save_exception(rb_eNoMemError, "Failed copying metrics entry");
+            return 0;
+        }
+        entry_hashmap_put(map, copy, copy);  // use the hashmap like hashset actually
+    }
+    return 1;
+}
+
+inline entry_t entry_new(buffer_t *source, uint32_t pos, uint32_t encoded_len, file_t *file_info) {
+    entry_t entry = (entry_t){.json = source->buffer + pos,
+                              .json_size = encoded_len,
+                              .pid = file_info->pid,
+                              .multiprocess_mode = file_info->multiprocess_mode,
+                              .type = file_info->type,
+                              .name_len = 0};
+
+    uint32_t value_offset = encoded_len + padding_length(encoded_len);
+    memcpy(&entry.value, entry.json + value_offset, sizeof(double));
+
+    return entry;
+}
+
+static int add_parsed_name(entry_t *entry) {
+    jsmn_parser parser;
+    jsmn_init(&parser);
+
+    jsmntok_t t[2] = {(jsmntok_t){}, (jsmntok_t){}};
+    jsmn_parse(&parser, entry->json, entry->json_size, t, 2);
+
+    jsmntok_t *name_tok = &t[1];
+
+    if (name_tok->start < name_tok->end && name_tok->start > 0) {
+        entry->name = entry->json + name_tok->start;
+        entry->name_len = name_tok->end - name_tok->start;
+        return 1;
+    }
+    return 0;
+}
+
+static int entry_lexical_comparator(const entry_t **a, const entry_t **b) {
+    size_t min_length = min((*a)->json_size, (*b)->json_size);
+    return strncmp((*a)->json, (*b)->json, min_length);
+}
+
+void hashmap_setup(struct hashmap *map) {
+    hashmap_init(map, (size_t(*)(const void *))hashmap_hash_entry,
+                 (int (*)(const void *, const void *))hashmap_compare_entry, 1000);
+}
+
+void entries_destroy(struct hashmap *map) {
+    struct hashmap_iter *iter;
+    for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
+        entry_t *entry = (entry_t *)entry_hashmap_iter_get_key(iter);
+        entry_free(entry);
+    }
+}
+
+int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map) {
+    if (source->size < START_POSITION) {
+        // nothing to read
+        return 1;[
+    }
+    uint32_t used;
+    memcpy(&used, source->buffer, sizeof(uint32_t));
+
+    if (used > source->size) {
+        save_exception(prom_eParsingError, "source file %s corrupted, used %u > file size %u", file_info->path, used,
+                       source->size);
+        return 0;
+    }
+
+    uint32_t pos = START_POSITION;
+    while (pos + sizeof(uint32_t) < used) {
+        uint32_t encoded_len;
+        memcpy(&encoded_len, source->buffer + pos, sizeof(uint32_t));
+        pos += sizeof(uint32_t);
+
+        uint32_t value_offset = encoded_len + padding_length(encoded_len);
+
+        if (pos + value_offset + sizeof(double) > used) {
+            save_exception(prom_eParsingError, "source file %s corrupted, used %u < stored data length %u",
+                           file_info->path, used, pos + value_offset + sizeof(double));
+            return 0;
+        }
+        entry_t entry = entry_new(source, pos, encoded_len, file_info);
+
+        if (!process_entry(map, &entry)) {
+            entries_destroy(map);
+            return 0;
+        }
+
+        pos += value_offset + sizeof(double);
+    }
+    return 1;
+}
+
+int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries) {
+    size_t num = hashmap_size(map);
+
+    size_t list_size = num * sizeof(entry_t *);
+    entry_t **list = malloc(list_size);
+
+    if (list == NULL) {
+        save_exception(rb_eNoMemError, "Couldn't allocate %zu memory", list_size);
+        return 0;
+    }
+
+    size_t cnt = 0;
+    struct hashmap_iter *iter;
+    for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
+        entry_t *entry = (entry_t *)entry_hashmap_iter_get_key(iter);
+        if (add_parsed_name(entry)) {
+            list[cnt] = entry;
+            cnt++;
+        }
+    }
+    if (cnt != num) {
+        save_exception(rb_eRuntimeError, "Processed entries %zu != map entries %zu", cnt, num);
+        free(list);
+        return 0;
+    }
+
+    qsort(list, cnt, sizeof(entry_t *), (compare_fn)&entry_lexical_comparator);
+    *sorted_entries = list;
+    return 1;
+}
+
+int is_pid_significant(const entry_t *e) {
+    ID mp = e->multiprocess_mode;
+    return e->type == sym_gauge && !(mp == sym_min || mp == sym_max || mp == sym_livesum);
+}

data/ext/fast_mmaped_file/#value_access.c#

@@ -0,0 +1,203 @@
+#include <ruby.h>
+#include <ruby/intern.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "file_format.h"
+#include "mmap.h"
+#include "value_access.h"
+
+static int open_and_extend_file(mm_ipc *i_mm, size_t len) {
+    int fd;
+
+    if ((fd = open(i_mm->t->path, i_mm->t->smode)) == -1) {
+        rb_raise(rb_eArgError, "Can't open %s", i_mm->t->path);
+    }
+
+    if (lseek(fd, len - i_mm->t->len - 1, SEEK_END) == -1) {
+        close(fd);
+        rb_raise(rb_eIOError, "Can't lseek %zu", len - i_mm->t->len - 1);
+    }
+
+    if (write(fd, "\000", 1) != 1) {
+        close(fd);
+        rb_raise(rb_eIOError, "Can't extend %s", i_mm->t->path);
+    }
+
+    return fd;
+}
+
+static void expand(mm_ipc *i_mm, size_t len) {
+    if (len < i_mm->t->len) {
+        rb_raise(rb_eArgError, "Can't reduce the size of mmap");
+    }
+
+    if (munmap(i_mm->t->addr, i_mm->t->len)) {
+        rb_raise(rb_eArgError, "munmap failed");
+    }
+
+    int fd = open_and_extend_file(i_mm, len);
+
+    i_mm->t->addr = mmap(0, len, i_mm->t->pmode, i_mm->t->vscope, fd, i_mm->t->offset);
+
+    if (i_mm->t->addr == MAP_FAILED) {
+        close(fd);
+        rb_raise(rb_eArgError, "mmap failed");
+    }
+
+    if (close(fd) == -1) {
+        rb_raise(rb_eArgError, "Can't close %s", i_mm->t->path);
+    }
+
+    if ((i_mm->t->flag & MM_LOCK) && mlock(i_mm->t->addr, len) == -1) {
+        rb_raise(rb_eArgError, "mlock(%d)", errno);
+    }
+    i_mm->t->len = len;
+    i_mm->t->real = len;
+}
+
+static void save_entry(mm_ipc *i_mm, size_t offset, VALUE key, VALUE value) {
+    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
+
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    memcpy(pos, &key_length, sizeof(uint32_t));
+    pos += sizeof(uint32_t);
+
+    memmove(pos, StringValuePtr(key), key_length);
+    pos += key_length;
+
+    memset(pos, ' ', padding_length(key_length));  // TODO: considder padding with /0
+    pos += padding_length(key_length);
+
+    double val = NUM2DBL(value);
+    memcpy(pos, &val, sizeof(double));
+}
+
+static void save_value(mm_ipc *i_mm, VALUE _offset, VALUE value) {
+    Check_Type(_offset, T_FIXNUM);
+    size_t offset = NUM2UINT(_offset);
+    if ((i_mm->t->real + sizeof(double)) <= offset) {
+        rb_raise(rb_eIndexError, "offset %zu out of string", offset);
+    }
+
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
+    }
+
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    double val = NUM2DBL(value);
+    memcpy(pos, &val, sizeof(double));
+}
+
+static VALUE load_value(mm_ipc *i_mm, VALUE _offset) {
+    Check_Type(_offset, T_FIXNUM);
+    size_t offset = NUM2UINT(_offset);
+    if ((i_mm->t->real + sizeof(double)) <= offset) {
+        rb_raise(rb_eIndexError, "offset %zu out of string", offset);
+    }
+
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    double value;
+    memcpy(&value, pos, sizeof(double));
+    return DBL2NUM(value);
+}
+
+inline uint32_t load_used(mm_ipc *i_mm) {
+	uint32_t used = *((uint32_t *)i_mm->t->addr);
+
+    if (used == 0) {
+        used = START_POSITION;
+    }
+    return used;
+}
+
+inline void save_used(mm_ipc *i_mm, uint32_t used) { *((uint32_t *)i_mm->t->addr) = used; }
+
+static VALUE initialize_entry(mm_ipc *i_mm, VALUE positions, VALUE key, VALUE value) {
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
+    }
+
+    if (RSTRING_LEN(key) > INT32_MAX) {
+        rb_raise(rb_eArgError, "string length gt %d", INT32_MAX);
+    }
+
+    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
+    uint32_t value_offset = sizeof(uint32_t) + key_length + padding_length(key_length);
+    uint32_t entry_length = value_offset + sizeof(double);
+
+    uint32_t used = load_used(i_mm);
+    while (i_mm->t->len < (used + entry_length)) {
+        expand(i_mm, i_mm->t->len * 2);
+    }
+    save_entry(i_mm, used, key, value);
+    save_used(i_mm, used + entry_length);
+
+    return rb_hash_aset(positions, key, INT2NUM(used + value_offset));
+}
+
+VALUE method_fetch_entry(VALUE self, VALUE positions, VALUE key, VALUE default_value) {
+    Check_Type(positions, T_HASH);
+    Check_Type(key, T_STRING);
+
+    mm_ipc *i_mm;
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    VALUE position = rb_hash_lookup(positions, key);
+
+    if (position != Qnil) {
+        return load_value(i_mm, position);
+    }
+
+    position = initialize_entry(i_mm, positions, key, default_value);
+    return load_value(i_mm, position);
+}
+
+VALUE method_upsert_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
+    Check_Type(positions, T_HASH);
+    Check_Type(key, T_STRING);
+
+    mm_ipc *i_mm;
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    VALUE position = rb_hash_lookup(positions, key);
+
+    if (position != Qnil) {
+        save_value(i_mm, position, value);
+        return load_value(i_mm, position);
+    }
+
+    position = initialize_entry(i_mm, positions, key, value);
+    return load_value(i_mm, position);
+}
+
+VALUE method_load_used(VALUE self) {
+    mm_ipc *i_mm;
+
+    GET_MMAP(self, i_mm, MM_MODIFY);
+    return UINT2NUM(load_used(i_mm));
+}
+
+VALUE method_save_used(VALUE self, VALUE value) {
+    Check_Type(value, T_FIXNUM);
+    mm_ipc *i_mm;
+
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
+    }
+
+    if (i_mm->t->len < INITIAL_SIZE) {
+        expand(i_mm, INITIAL_SIZE);
+    }
+
+    save_used(i_mm, NUM2UINT(value));
+    return value;
+}

data/lib/prometheus/client.rb

@@ -29,6 +29,11 @@ module Prometheus
         configuration.pid_provider.call
       end
 
+      def reset!
+        @registry = nil
+        ::Prometheus::Client::MmapedValue.reset_and_reinitialize
+      end
+
       def reinitialize_on_pid_change
         ::Prometheus::Client::MmapedValue.reinitialize_on_pid_change
       end

data/lib/prometheus/client/#histogram.rb#

@@ -0,0 +1,79 @@
+ip route get 1require 'prometheus/client/metric'
+require 'prometheus/client/uses_value_type'
+
+module Prometheus
+  module Client
+    # A histogram samples observations (usually things like request durations
+    # or response sizes) and counts them in configurable buckets. It also
+    # provides a sum of all observed values.
+    class Histogram < Metric
+      # Value represents the state of a Histogram at a given point.
+      class Value < Hash
+        include UsesValueType
+        attr_accessor :sum, :total
+
+        def initialize(type, name, labels, buckets)
+          @sum = value_object(type, name, "#{name}_sum", labels)
+          # TODO: get rid of total and use +Inf bucket instead.
+          @total = value_object(type, name, "#{name}_count", labels)
+
+          buckets.each do |bucket|
+            self[bucket] = value_object(type, name, "#{name}_bucket", labels.merge({ :le => bucket.to_s }))
+          end
+        end
+
+        def observe(value)
+          @sum.increment(value)
+          @total.increment()
+
+          each_key do |bucket|
+            self[bucket].increment() if value <= bucket
+          end
+        end
+
+        def get()
+          hash = {}
+          each_key do |bucket|
+            hash[bucket] = self[bucket].get()
+          end
+          hash
+        end
+      end
+
+      # DEFAULT_BUCKETS are the default Histogram buckets. The default buckets
+      # are tailored to broadly measure the response time (in seconds) of a
+      # network service. (From DefBuckets client_golang)
+      DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1,
+                         2.5, 5, 10].freeze
+
+      # Offer a way to manually specify buckets
+      def initialize(name, docstring, base_labels = {},
+                     buckets = DEFAULT_BUCKETS)
+        raise ArgumentError, 'Unsorted buckets, typo?' unless sorted? buckets
+
+        @buckets = buckets
+        super(name, docstring, base_labels)
+      end
+
+      def type
+        :histogram
+      end
+
+      def observe(labels, value)
+        label_set = label_set_for(labels)
+        synchronize { @values[label_set].observe(value) }
+      end
+
+      private
+
+      def default(labels)
+        # TODO: default function needs to know key of hash info (label names and values)
+        Value.new(type, @name, labels, @buckets)
+      end
+
+      def sorted?(bucket)
+        bucket.each_cons(2).all? { |i, j| i <= j }
+      end
+    end
+  end
+end