prometheus-client-mmap 0.7.0.beta14 → 0.7.0.beta15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed314a3faf8c4af821870d160f43519e49dd7ebd
-  data.tar.gz: cf39eae7140111a9216475d159597dbe86c21384
+  metadata.gz: 84f31b1815d567cd2d00583255372973f739fad7
+  data.tar.gz: b2752140fb46a42db2081099a42943d33dc890d5
 SHA512:
-  metadata.gz: 7ccd3d06b4635af116a2538c3519bf63a5b5c939cbdbb6d29414e61635162e0d099afb926b9a1d462f2f8946b4f6800f9da88819e10ebb05e7055f3d6e53e02a
-  data.tar.gz: 4a4ed88916f09e41acbb0ed6193e4b767e4f0d03ec4461305332b401018dd847ba503bc699c145916856284c331921e8cab3495c5367412b882f58af129028c2
+  metadata.gz: 8b2e8c8b8ec3ea087da3d13e142c24622b56c007aa8679dc18beb4bd1ae66a28ef72344bcfa531c39e2c5778d83e3ba563b56cdcd49dfb811b4e6b34ace2f99d
+  data.tar.gz: 71488cc0a7a41f9c0cc230c11305d654f813b94502d440dd37ee98e79cf43feba70959ee9985a172b8778226de8035a78dac29d45e19f6a220b8a93ad33fe866

data/lib/prometheus/client/formats/text.rb

@@ -96,6 +96,7 @@ module Prometheus
             metrics = {}
             Dir.glob(File.join(path, '*.db')).sort.each do |f|
               parts = File.basename(f, '.db').split('_')
+              parts[-1].gsub!(/(?<=.)-\d+/, '') # remove trailing file number
               type = parts[0].to_sym
 
               MmapedDict.read_all_values(f).each do |key, value|

data/lib/prometheus/client/helper/mmaped_file.rb

@@ -0,0 +1,150 @@
+require 'prometheus/client'
+require 'mmap'
+
+module Prometheus
+  module Client
+    module Helper
+      class MmapedFile < Mmap
+        MINIMUM_SIZE = 8
+        attr_reader :filepath, :size
+
+        def initialize(filepath, mode = 'r', protection = Mmap::MAP_SHARED, options = {})
+          @filepath = filepath
+
+          File.open(filepath, 'a+b') do |file|
+            file.truncate(initial_mmap_file_size) if file.size < MINIMUM_SIZE
+            @size = file.size
+          end
+
+          super(filepath, mode, protection, options)
+        end
+
+        def pid
+          self[0..3].unpack('l')[0]
+        end
+
+        def pid=(value)
+          self[0..3] = [value].pack('l')
+        end
+
+        def used
+          self[4..7].unpack('l')[0]
+        end
+
+        def used=(value)
+          self[4..7] = [value].pack('l')
+        end
+
+        def locked_to_process?
+          pid > 0
+        end
+
+        def lock_owner?
+          pid == Process.pid
+        end
+
+        def lock_to_process
+          return true if lock_owner?
+
+          if locked_to_process?
+            false
+          else
+            self.pid = Process.pid
+
+            # check if PID was correctly written
+            lock_owner?
+          end
+        end
+
+        def unlock
+          return unless lock_owner?
+
+          self.pid = 0
+        end
+
+        def initial_mmap_file_size
+          Prometheus::Client.configuration.initial_mmap_file_size
+        end
+
+        def entries
+          return Enumerator.new {} if used.zero?
+
+          Enumerator.new do |yielder|
+            used_ = used # cache used to avoid unnecessary unpack operations
+
+            pos = 12 # pid + used + padding offset
+            while pos < used_
+              data = slice(pos..-1)
+              encoded_len, = data.unpack('l')
+              padding_len = 8 - (encoded_len + 4) % 8
+              value_offset = 4 + encoded_len + padding_len
+
+              yielder.yield data, encoded_len, value_offset
+
+              pos += value_offset + 8
+            end
+          end
+        end
+
+        def add_entry(data, value)
+          self.used = 12 if used.zero?
+
+          # Pad to be 8-byte aligned.
+          padded = data + (' ' * (8 - (data.length + 4) % 8))
+          entry = [data.length, padded, value].pack("lA#{padded.length}d")
+          used_ = used
+
+          while (used_ + entry.length) > @size do
+            extend(size)
+            @size = File.size(filepath)
+          end
+
+          self[used_..used_ + entry.length] = entry
+
+          self.used = used_ + entry.length
+        end
+
+        def close
+          munmap
+        end
+
+        class << self
+          def open_readonly(filepath)
+            MmapedFile.new(filepath, 'r', Mmap::MAP_PRIVATE)
+          end
+
+          def open(filepath)
+            MmapedFile.new(filepath, 'rw', Mmap::MAP_SHARED)
+          end
+
+          def lock_to_process(filepath)
+            m = open(filepath)
+            if m.lock_to_process
+              Kernel.at_exit do
+                open(filepath).unlock
+              end
+            end
+          ensure
+            m.close
+          end
+
+          def ensure_process_exclusive_file(file_prefix = 'mmaped_file')
+            path = nil
+            filename_number = 0
+            until path && Helper::MmapedFile.lock_to_process(path)
+              filename = "#{file_prefix}_#{Prometheus::Client.pid}-#{filename_number}.db"
+              path = File.join(Prometheus::Client.configuration.multiprocess_files_dir, filename)
+              filename_number += 1
+            end
+            path
+          end
+
+          def open_process_exclusive_file(file_prefix = 'mmaped_file')
+            filename = Helper::MmapedFile.ensure_process_exclusive_file(file_prefix)
+            open(filename)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prometheus/client/mmaped_dict.rb

@@ -1,5 +1,5 @@
+require 'prometheus/client/helper/mmaped_file'
 require 'prometheus/client'
-require 'mmap'
 
 module Prometheus
   module Client
@@ -13,36 +13,34 @@ module Prometheus
     # There's then a number of entries, consisting of a 4 byte int which is the
     # size of the next field, a utf-8 encoded string key, padding to an 8 byte
     # alignment, and then a 8 byte float which is the value.
-    #
-    # TODO(julius): dealing with Mmap.new, truncate etc. errors?
     class MmapedDict
-      MINIMUM_SIZE = 4.freeze
-      attr_reader :m, :capacity, :used, :positions
+      MINIMUM_SIZE = 8
+      attr_reader :m, :used, :positions
 
-      def initialize(filename)
+      def initialize(m)
         @mutex = Mutex.new
-        @f = File.open(filename, 'a+b')
-        process_file
+
+        @m = m
+        # @m.mlock # TODO: Ensure memory is locked to RAM
+
+        @used = @m.used
+
+        @positions = {}
+        read_all_positions.each do |key, _, pos|
+          @positions[key] = pos
+        end
       rescue StandardError => e
-        raise ParsingError, "exception #{e} while processing metrics file #{@f.path}"
+        raise ParsingError, "exception #{e} while processing metrics file #{path}"
       end
 
       # Yield (key, value). No locking is performed.
       def self.read_all_values(f)
-        m = Mmap.new(f, 'rw', Mmap::MAP_SHARED)
-        used, = m[0..3].unpack('l')
-        pos = 8
-        values = []
-        while pos < used
-          data = m.slice(pos..-1)
-          encoded_len, = data.unpack('l')
-          value_offset = 4 + encoded_len + (8 - (encoded_len + 4) % 8)
+        m = Helper::MmapedFile.open(f)
 
+        m.entries.map do |data, encoded_len, value_offset|
           encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset))
-          values << [encoded, value]
-          pos += value_offset + 8
+          [encoded, value]
         end
-        values
       ensure
         m.munmap
       end
@@ -66,15 +64,13 @@ module Prometheus
       end
 
       def path
-        @f.path unless @f.nil?
+        @m.filepath unless @m.nil?
       end
 
       def close
-        @m.munmap
+        @m.close
       rescue TypeError => e
         Prometheus::Client.logger.warn("munmap raised error #{e}")
-      ensure
-        @f.close
       end
 
       def initial_mmap_file_size
@@ -83,58 +79,22 @@ module Prometheus
 
       private
 
-      def process_file
-        @f.truncate(initial_mmap_file_size) if @f.size < MINIMUM_SIZE
-
-        @capacity = @f.size
-        @m = Mmap.new(@f.path, 'rw', Mmap::MAP_SHARED)
-        # @m.mlock # TODO: Why does this raise an error?
-
-        @positions = {}
-        @used = @m[0..3].unpack('l')[0]
-        if @used == 0
-          @used = 8
-          @m[0..3] = [@used].pack('l')
-        else
-          read_all_positions.each do |key, _, pos|
-            @positions[key] = pos
-          end
-        end
-      end
-
       # Initialize a value. Lock must be held by caller.
       def init_value(key)
-        # Pad to be 8-byte aligned.
-        padded = key + (' ' * (8 - (key.length + 4) % 8))
-        value = [key.length, padded, 0.0].pack("lA#{padded.length}d")
-        while @used + value.length > @capacity
-          @capacity *= 2
-          @f.truncate(@capacity)
-          @m.unmap
-          @m = Mmap.new(@f.path, 'rw', Mmap::MAP_SHARED)
-        end
-        @m[@used..@used + value.length] = value
+        @m.add_entry(key, 0.0)
 
         # Update how much space we've used.
-        @used += value.length
-        @m[0..3] = [@used].pack('l')
+        @used = @m.used
+
         @positions[key] = @used - 8
       end
 
       # Yield (key, value, pos). No locking is performed.
       def read_all_positions
-        pos = 8
-        values = []
-        while pos < @used
-          data = @m.slice(pos..-1)
-          encoded_len = data.unpack('l')[0]
-          padding_len = 8 - (encoded_len + 4) % 8
-          encoded = data.unpack(format('@4A%d', encoded_len))
-          pos += 4 + encoded_len + padding_len
-          values << [encoded, pos]
-          pos += 8
+        @m.entries.map do |data, encoded_len, value_offset|
+          encoded, = data.unpack(format('@4A%d', encoded_len))
+          [encoded, value_offset]
         end
-        values
       end
     end
   end

data/lib/prometheus/client/mmaped_value.rb

@@ -81,12 +81,11 @@ module Prometheus
         @@files_lock.synchronize do
           unless @@files.has_key?(@file_prefix)
             unless @file.nil?
-              puts @file
               @file.close
             end
+            mmaped_file = Helper::MmapedFile.open_process_exclusive_file(@file_prefix)
 
-            filename = File.join(Prometheus::Client.configuration.multiprocess_files_dir, "#{@file_prefix}_#{@@pid}.db")
-            @@files[@file_prefix] = MmapedDict.new(filename)
+            @@files[@file_prefix] = MmapedDict.new(mmaped_file)
           end
         end
 
@@ -110,12 +109,15 @@ module Prometheus
         @file.write_value(key, val)
       rescue StandardError => e
         Prometheus::Client.logger.warn("writing value to #{@file.path} failed with #{e}")
+        Prometheus::Client.logger.debug(e.backtrace.join("\n"))
       end
 
       def read_value(key)
         @file.read_value(key)
       rescue StandardError => e
         Prometheus::Client.logger.warn("reading value from #{@file.path} failed with #{e}")
+        Prometheus::Client.logger.debug(e.backtrace.join("\n"))
+        0
       end
     end
   end

data/lib/prometheus/client/simple_value.rb

@@ -4,7 +4,7 @@ require 'mmap'
 module Prometheus
   module Client
     class SimpleValue
-      def initialize(type, metric_name, name, labels, value = 0)
+      def initialize(type, metric_name, name, labels, value = 0.0)
         @value = value
       end
 

data/lib/prometheus/client/summary.rb

@@ -46,7 +46,7 @@ module Prometheus
         @validator.valid?(labels)
 
         synchronize do
-          @values[labels].sum
+          @values[labels].sum.get
         end
       end
 

data/lib/prometheus/client/support/unicorn.rb

@@ -4,9 +4,12 @@ module Prometheus
       module Unicorn
         def self.worker_pid_provider
           wid = worker_id
-          Process.pid if wid.nil?
-
-          "worker_id_#{wid}"
+          wid = Process.pid if wid.nil?
+          if wid.nil?
+            "process_pid_#{Process.pid}"
+          else
+            "worker_id_#{wid}"
+          end
         end
 
         def self.worker_id

data/lib/prometheus/client/uses_value_type.rb

@@ -11,7 +11,7 @@ module Prometheus
       def value_object(type, metric_name, name, labels, *args)
         value_class.new(type, metric_name, name, labels, *args)
       rescue StandardError => e
-        Prometheus::Client.logger.info("error #{e} while creating instance of #{value_class} defaultig to SimpleValue")
+        Prometheus::Client.logger.info("error #{e} while creating instance of #{value_class} defaulting to SimpleValue")
         Prometheus::Client::SimpleValue.new(type, metric_name, name, labels)
       end
     end

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '0.7.0.beta14'
+    VERSION = '0.7.0.beta15'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 0.7.0.beta14
+  version: 0.7.0.beta15
 platform: ruby
 authors:
 - Tobias Schmidt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-29 00:00:00.000000000 Z
+date: 2017-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mmap2
@@ -45,6 +45,7 @@ files:
 - lib/prometheus/client/formats/text.rb
 - lib/prometheus/client/gauge.rb
 - lib/prometheus/client/helper/json_parser.rb
+- lib/prometheus/client/helper/mmaped_file.rb
 - lib/prometheus/client/histogram.rb
 - lib/prometheus/client/label_set_validator.rb
 - lib/prometheus/client/metric.rb