prometheus-client-mmap 0.7.0.beta34 → 0.7.0.beta35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1225b344f2939116817abcefbfe4483e38c7087c
-  data.tar.gz: 4fe5ade968d6d4df62d7ab7b9801ee8b1c80aa50
+  metadata.gz: d8a992632af121ceb8b2b8049f293e69822afbce
+  data.tar.gz: 99c14fe2ced8279e6f02e9ca13fef3293bf55090
 SHA512:
-  metadata.gz: f3afd81bf5703647413b61a0762882bc3796325a51fa0db271620d6b0c1d70bbcba0d5191c9acf740e2d856539982a04bdcc253c4f4c1ecb969e9affca7dcb73
-  data.tar.gz: f083690e3b0e19f3968576c64d7f31e71ffa982b90f29edfb5e8ce6b738f2e97a5345f6dfc2f23ed98dd6a04b0d06f6015fb377b519bd760d1bf9cc3b796ca7a
+  metadata.gz: 712cdf6271eb5660c4d13852be75a70a9aab6c736a9e8f77588e18ccb0ce82c22001e146273ee2ae4fa95270ea4bd3fe562541b5d25ed9e3bd49033724cc1dea
+  data.tar.gz: f49a4b52b8d5dcc2c629c52ab010dc979d8401c928632d51a0d82f211fadd9870503c9e4a7aa5cea5d6aab93fad76d6fa934eba76eb6f5d6b2072809be656444

data/README.md

@@ -10,7 +10,7 @@ through a HTTP interface. Intended to be used together with a
 
 [![Gem Version][4]](http://badge.fury.io/rb/prometheus-client-mmap)
 [![Build Status][3]](https://gitlab.com/gitlab-org/prometheus-client-mmap/commits/master)
-[![Dependency Status][5]](https://gemnasium.com/prometheus/client_ruby)
+[![Dependency Status][5]](https://gemnasium.com/prometheus/prometheus-client-mmap)
 
 ## Usage
 
@@ -185,6 +185,23 @@ To use it add this line to your `configure` block:
   config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider)
 ```
 
+## Tools
+ 
+###`bin/parse`
+
+This command can be used to parse metric files located on the filesystem just like a metric exporter would.
+It outputs either `json` formatted raw data or digested data in prometheus `text` format.
+
+#### Usage:
+
+```bash
+$ ./bin/parse -h
+Usage: parse [options] files...
+    -t, --to-prometheus-text         format output using Prometheus text formatter
+    -p, --profile                    enable profiling
+    -h, --help                       Show this message
+```
+
 ## Tests
 
 Install necessary development gems with `bundle install` and run tests with

data/ext/fast_mmaped_file/fast_mmaped_file.c

@@ -12,33 +12,48 @@ VALUE MMAPED_FILE = Qnil;
 #define START_POSITION 8
 #define INITIAL_SIZE (2 * sizeof(int32_t))
 
-void expand(mm_ipc *i_mm, size_t len){
+int open_and_extend_file(mm_ipc *i_mm, size_t len) {
     int fd;
-    if (munmap(i_mm->t->addr, i_mm->t->len)) {
-        rb_raise(rb_eArgError, "munmap failed");
-    }
+
     if ((fd = open(i_mm->t->path, i_mm->t->smode)) == -1) {
         rb_raise(rb_eArgError, "Can't open %s", i_mm->t->path);
     }
 
-    if (len < i_mm->t->len) {
-        rb_raise(rb_eArgError, "Can't reduce the size of mmap");
-    }
-
     if (lseek(fd, len - i_mm->t->len - 1, SEEK_END) == -1) {
+        close(fd);
         rb_raise(rb_eIOError, "Can't lseek %lu", len - i_mm->t->len - 1);
     }
+
     if (write(fd, "\000", 1) != 1) {
+        close(fd);
         rb_raise(rb_eIOError, "Can't extend %s", i_mm->t->path);
     }
 
+    return fd;
+}
+
+void expand(mm_ipc *i_mm, size_t len) {
+    if (len < i_mm->t->len) {
+        rb_raise(rb_eArgError, "Can't reduce the size of mmap");
+    }
+
+    if (munmap(i_mm->t->addr, i_mm->t->len)) {
+        rb_raise(rb_eArgError, "munmap failed");
+    }
+
+    int fd = open_and_extend_file(i_mm, len);
+
     i_mm->t->addr = mmap(0, len, i_mm->t->pmode, i_mm->t->vscope, fd, i_mm->t->offset);
-    close(fd);
 
     if (i_mm->t->addr == MAP_FAILED) {
+        close(fd);
         rb_raise(rb_eArgError, "mmap failed");
     }
 
+    if (close(fd) == -1){
+        rb_raise(rb_eArgError, "Can't close %s", i_mm->t->path);
+    }
+
     if ((i_mm->t->flag & MM_LOCK) && mlock(i_mm->t->addr, len) == -1) {
         rb_raise(rb_eArgError, "mlock(%d)", errno);
     }
@@ -46,8 +61,8 @@ void expand(mm_ipc *i_mm, size_t len){
     i_mm->t->real = len;
 }
 
-inline uint32_t padding_length(uint32_t key_length){
-    return 8 - (key_length + 4) % 8; // padding | 8 byte aligned
+inline uint32_t padding_length(uint32_t key_length) {
+    return 8 - (sizeof(uint32_t) + key_length) % 8; // padding | 8 byte aligned
 }
 
 void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
@@ -68,9 +83,8 @@ void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
     memcpy(pos, &val, sizeof(double));
 }
 
-inline uint32_t load_used(mm_ipc *i_mm){
-    uint32_t used = 0;
-    memcpy(&used, (char *)i_mm->t->addr, sizeof(uint32_t));
+inline uint32_t load_used(mm_ipc *i_mm) {
+    uint32_t used = *((uint32_t *)i_mm->t->addr);
 
     if (used == 0){
         used = START_POSITION;
@@ -78,8 +92,29 @@ inline uint32_t load_used(mm_ipc *i_mm){
     return used;
 }
 
-inline void save_used(mm_ipc *i_mm, uint32_t used){
-    memcpy((char *)i_mm->t->addr, &used, sizeof(uint32_t));
+inline void save_used(mm_ipc *i_mm, uint32_t used) {
+    *((uint32_t *)i_mm->t->addr) = used;
+}
+
+VALUE method_load_used(VALUE self) {
+    mm_ipc *i_mm;
+
+    GET_MMAP(self, i_mm, MM_MODIFY);
+    return UINT2NUM(load_used(i_mm));
+}
+
+VALUE method_save_used(VALUE self, VALUE value) {
+	Check_Type(value, T_FIXNUM);
+    mm_ipc *i_mm;
+
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    if (i_mm->t->len < INITIAL_SIZE) {
+        expand(i_mm, INITIAL_SIZE);
+    }
+
+    save_used(i_mm, NUM2UINT(value));
+    return value;
 }
 
 VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
@@ -92,17 +127,13 @@ VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
     }
 
     mm_ipc *i_mm;
-    GetMmap(self, i_mm, MM_MODIFY);
-
-    if ((i_mm->t->len) < INITIAL_SIZE) {
-        expand(i_mm, INITIAL_SIZE);
-    }
+    GET_MMAP(self, i_mm, MM_MODIFY);
 
-    if (i_mm->t->flag & MM_FROZEN){
+    if (i_mm->t->flag & MM_FROZEN) {
         rb_error_frozen("mmap");
     }
 
-    if (RSTRING_LEN(key) > UINT32_MAX){
+    if (RSTRING_LEN(key) > UINT32_MAX) {
         rb_raise(rb_eArgError, "string length gt %u", UINT32_MAX);
     }
 
@@ -122,7 +153,7 @@ VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
 
 VALUE method_get_double(VALUE self, VALUE index) {
     mm_ipc *i_mm;
-    GetMmap(self, i_mm, MM_MODIFY);
+    GET_MMAP(self, i_mm, MM_MODIFY);
 
     Check_Type(index, T_FIXNUM);
     size_t idx = NUM2UINT(index);
@@ -140,5 +171,7 @@ VALUE method_get_double(VALUE self, VALUE index) {
 void Init_fast_mmaped_file() {
     MMAPED_FILE = rb_define_module("FastMmapedFile");
     rb_define_method(MMAPED_FILE, "get_double", method_get_double, 1);
+    rb_define_method(MMAPED_FILE, "used", method_load_used, 0);
+    rb_define_method(MMAPED_FILE, "used=", method_save_used, 1);
     rb_define_method(MMAPED_FILE, "add_entry", method_add_entry, 3);
 }

data/ext/fast_mmaped_file/mmap.h

@@ -45,7 +45,7 @@ typedef struct {
 } mm_ipc;
 
 
-#define GetMmap(obj, i_mm, t_modify)                             \
+#define GET_MMAP(obj, i_mm, t_modify)                             \
     Data_Get_Struct(obj, mm_ipc, i_mm);                          \
     if (!i_mm->t->path) {                                        \
         rb_raise(rb_eIOError, "unmapped file");                  \
@@ -54,5 +54,4 @@ typedef struct {
         rb_error_frozen("mmap");                                 \
     }
 
-
-#endif
+#endif

data/lib/prometheus/client/gauge.rb

@@ -27,6 +27,10 @@ module Prometheus
       def set(labels, value)
         @values[label_set_for(labels)].set(value)
       end
+
+      def increment(labels, value)
+        @values[label_set_for(labels)].increment(value)
+      end
     end
   end
 end

data/lib/prometheus/client/helper/entry_parser.rb

@@ -6,8 +6,11 @@ module Prometheus
       module EntryParser
         class ParsingError < RuntimeError;
         end
+
         MINIMUM_SIZE = 8
         START_POSITION = 8
+        VALUE_BYTES = 8
+        ENCODED_LENGTH_BYTES = 4
 
         def used
           slice(0..3).unpack('l')[0]
@@ -57,16 +60,18 @@ module Prometheus
                 next
               end
 
-              padding_len = 8 - (encoded_len + 4) % 8
-              value_offset = 4 + encoded_len + padding_len
+              entry_len = ENCODED_LENGTH_BYTES + encoded_len
+              padding_len = 8 - entry_len % 8
+
+              value_offset = entry_len + padding_len # align to 8 bytes
               pos += value_offset
 
-              if value_offset > 0 && (pos + 8) < size # if positions are safe
+              if value_offset > 0 && (pos + VALUE_BYTES) < size # if positions are safe
                 yielder.yield data, encoded_len, value_offset, pos
               else
                 raise ParsingError, "data slice is nil at pos #{pos}" unless ignore_errors
               end
-              pos += 8
+              pos += VALUE_BYTES
             end
           end
         end
@@ -82,7 +87,7 @@ module Prometheus
             end
           end
 
-          result.reject { |e| e.nil? } if ignore_errors
+          metrics.reject!(&:nil?) if ignore_errors
           result
         end
 
@@ -90,8 +95,9 @@ module Prometheus
           parsed_entries(ignore_errors).each do |key, value|
             begin
               metric_name, name, labelnames, labelvalues = JsonParser.load(key)
-              labelnames = [] unless labelnames
-              labelvalues = [] unless labelvalues
+              labelnames ||= []
+              labelvalues ||= []
+
               metric = metrics.fetch(metric_name,
                                      metric_name: metric_name,
                                      help: 'Multiprocess metric',
@@ -111,7 +117,7 @@ module Prometheus
             end
           end
 
-          metrics.reject { |e| e.nil? } if ignore_errors
+          metrics.reject!(&:nil?) if ignore_errors
           metrics
         end
 

data/lib/prometheus/client/helper/mmaped_file.rb

@@ -7,8 +7,9 @@ module Prometheus
   module Client
     module Helper
       class MmapedFile < Mmap
-        include FastMmapedFile
         include EntryParser
+        include FastMmapedFile
+
         attr_reader :filepath, :size
 
         def initialize(filepath, mode = 'r', protection = Mmap::MAP_SHARED, options = {})
@@ -22,10 +23,6 @@ module Prometheus
           super(filepath, mode, protection, options)
         end
 
-        def used=(value)
-          self[0..3] = [value].pack('l')
-        end
-
         def close
           munmap
         end

data/lib/prometheus/client/helper/plain_file.rb

@@ -3,6 +3,7 @@ require 'prometheus/client/helper/entry_parser'
 module Prometheus
   module Client
     module Helper
+      # Parses DB files without using mmap
       class PlainFile
         include EntryParser
         attr_reader :filepath

data/lib/prometheus/client/mmaped_value.rb

@@ -6,8 +6,9 @@ module Prometheus
   module Client
     # A float protected by a mutex backed by a per-process mmaped file.
     class MmapedValue
-      @@files = {}
       VALUE_LOCK = Mutex.new
+
+      @@files = {}
       @@pid = -1
 
       def initialize(type, metric_name, name, labels, multiprocess_mode = '')
@@ -25,7 +26,7 @@ module Prometheus
         initialize_file
       end
 
-      def increment(amount=1)
+      def increment(amount = 1)
         @mutex.synchronize do
           initialize_file if pid_changed?
 
@@ -56,35 +57,19 @@ module Prometheus
         @pid != Process.pid
       end
 
-      def reinitialize
-        initialize_file if pid_changed?
+      # method needs to be run in VALUE_LOCK mutex
+      def unsafe_reinitialize_file(check_pid = true)
+        unsafe_initialize_file if !check_pid || pid_changed?
       end
 
-      def initialize_file
+      def self.reset_and_reinitialize
         VALUE_LOCK.synchronize do
-          return if @file && !pid_changed?
-          self.class.reset_on_pid_change
-
-          @pid = Process.pid
-          unless @@files.has_key?(@file_prefix)
-            unless @file.nil?
-              @file.close
-            end
-            mmaped_file = Helper::MmapedFile.open_exclusive_file(@file_prefix)
-
-            @@files[@file_prefix] = MmapedDict.new(mmaped_file)
-          end
+          @@pid = Process.pid
+          @@files = {}
 
-          @file = @@files[@file_prefix]
-          labelnames = []
-          labelvalues = []
-          @labels.each do |k, v|
-            labelnames << k
-            labelvalues << v
+          ObjectSpace.each_object(MmapedValue).each do |v|
+            v.unsafe_reinitialize_file(false)
           end
-
-          @key = [@metric_name, @name, labelnames, labelvalues].to_json
-          @value = read_value(@key)
         end
       end
 
@@ -98,9 +83,9 @@ module Prometheus
       def self.reinitialize_on_pid_change
         VALUE_LOCK.synchronize do
           reset_on_pid_change
-        end
 
-        ObjectSpace.each_object(MmapedValue, &:reinitialize)
+          ObjectSpace.each_object(MmapedValue, &:unsafe_reinitialize_file)
+        end
       end
 
       def self.pid_changed?
@@ -113,9 +98,45 @@ module Prometheus
 
       private
 
+      def initialize_file
+        VALUE_LOCK.synchronize do
+          unsafe_initialize_file
+        end
+      end
+
+      def unsafe_initialize_file
+        self.class.reset_on_pid_change
+
+        @pid = Process.pid
+        unless @@files.has_key?(@file_prefix)
+          unless @file.nil?
+            @file.close
+          end
+          mmaped_file = Helper::MmapedFile.open_exclusive_file(@file_prefix)
+
+          @@files[@file_prefix] = MmapedDict.new(mmaped_file)
+        end
+
+        @file = @@files[@file_prefix]
+        @key = rebuild_key
+
+        @value = read_value(@key)
+      end
+
+
+      def rebuild_key
+        labelnames = []
+        labelvalues = []
+        @labels.each do |k, v|
+          labelnames << k
+          labelvalues << v
+        end
+
+        [@metric_name, @name, labelnames, labelvalues].to_json
+      end
+
       def write_value(key, val)
         @file.write_value(key, val)
-
       rescue StandardError => e
         Prometheus::Client.logger.warn("writing value to #{@file.path} failed with #{e}")
         Prometheus::Client.logger.debug(e.backtrace.join("\n"))

data/lib/prometheus/client/simple_value.rb

@@ -4,7 +4,7 @@ require 'mmap'
 module Prometheus
   module Client
     class SimpleValue
-      def initialize(type, metric_name, name, _labels, *_args)
+      def initialize(_type, _metric_name, _name, _labels, *_args)
         @value = 0.0
       end
 

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '0.7.0.beta34'.freeze
+    VERSION = '0.7.0.beta35'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 0.7.0.beta34
+  version: 0.7.0.beta35
 platform: ruby
 authors:
 - Tobias Schmidt