prometheus-client-mmap 0.7.0.beta41 → 0.7.0.beta42

data/ext/fast_mmaped_file/rendering.h

@@ -3,6 +3,6 @@
 #include <ruby.h>
 #include <file_parsing.h>
 
-VALUE entries_to_string(entry_struct **sorted_entries, size_t entries_count);
+int entries_to_string(VALUE string, entry_t **sorted_entries, size_t entries_count);
 
-#endif
+#endif

data/ext/fast_mmaped_file/utils.c

@@ -1,7 +1,8 @@
 #include <ruby.h>
+
 #include "utils.h"
 
-void save_exception(VALUE exception, const char *fmt, ...){
+void save_exception(VALUE exception, const char *fmt, ...) {
     va_list args;
 
     va_start(args, fmt);
@@ -13,7 +14,7 @@ void save_exception(VALUE exception, const char *fmt, ...){
     rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception_message"), message);
 }
 
-void raise_last_exception(){
+NORETURN(void raise_last_exception()) {
     VALUE current_thread = rb_thread_current();
     VALUE exception = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception"));
     VALUE message = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception_message"));

data/ext/fast_mmaped_file/utils.h

@@ -20,7 +20,7 @@
        __typeof__ (b) _b = (b); \
      _a > _b ? _a : _b; })
 
-void raise_last_exception();
+NORETURN(void raise_last_exception());
 void save_exception(VALUE exception, const char *fmt, ...);
 
 #endif

data/ext/fast_mmaped_file/value_access.c

@@ -1,16 +1,16 @@
 #include <ruby.h>
 #include <ruby/intern.h>
 
+#include <errno.h>
+#include <fcntl.h>
 #include <sys/mman.h>
 #include <unistd.h>
-#include <fcntl.h>
-#include <errno.h>
 
 #include "file_format.h"
 #include "mmap.h"
 #include "value_access.h"
 
-int open_and_extend_file(mm_ipc *i_mm, size_t len) {
+static int open_and_extend_file(mm_ipc *i_mm, size_t len) {
     int fd;
 
     if ((fd = open(i_mm->t->path, i_mm->t->smode)) == -1) {
@@ -30,7 +30,7 @@ int open_and_extend_file(mm_ipc *i_mm, size_t len) {
     return fd;
 }
 
-void expand(mm_ipc *i_mm, size_t len) {
+static void expand(mm_ipc *i_mm, size_t len) {
     if (len < i_mm->t->len) {
         rb_raise(rb_eArgError, "Can't reduce the size of mmap");
     }
@@ -59,7 +59,7 @@ void expand(mm_ipc *i_mm, size_t len) {
     i_mm->t->real = len;
 }
 
-void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
+static void save_entry(mm_ipc *i_mm, size_t offset, VALUE key, VALUE value) {
     uint32_t key_length = (uint32_t)RSTRING_LEN(key);
 
     char *pos = (char *)i_mm->t->addr + offset;
@@ -70,59 +70,56 @@ void save_entry(mm_ipc *i_mm, uint32_t offset, VALUE key, VALUE value){
     memmove(pos, StringValuePtr(key), key_length);
     pos += key_length;
 
-    memset(pos, ' ', padding_length(key_length));
+    memset(pos, ' ', padding_length(key_length));  // TODO: considder padding with /0
     pos += padding_length(key_length);
 
     double val = NUM2DBL(value);
     memcpy(pos, &val, sizeof(double));
 }
 
-inline uint32_t load_used(mm_ipc *i_mm) {
-    uint32_t used = *((uint32_t *)i_mm->t->addr);
-
-    if (used == 0) {
-        used = START_POSITION;
+static void save_value(mm_ipc *i_mm, VALUE _offset, VALUE value) {
+    Check_Type(_offset, T_FIXNUM);
+    size_t offset = NUM2UINT(_offset);
+    if ((i_mm->t->real + sizeof(double)) <= offset) {
+        rb_raise(rb_eIndexError, "offset %zu out of string", offset);
     }
-    return used;
-}
 
-inline void save_used(mm_ipc *i_mm, uint32_t used) {
-    *((uint32_t *)i_mm->t->addr) = used;
-}
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
+    }
 
-VALUE method_load_used(VALUE self) {
-    mm_ipc *i_mm;
+    char *pos = (char *)i_mm->t->addr + offset;
 
-    GET_MMAP(self, i_mm, MM_MODIFY);
-    return UINT2NUM(load_used(i_mm));
+    double val = NUM2DBL(value);
+    memcpy(pos, &val, sizeof(double));
 }
 
-VALUE method_save_used(VALUE self, VALUE value) {
-    Check_Type(value, T_FIXNUM);
-    mm_ipc *i_mm;
-
-    GET_MMAP(self, i_mm, MM_MODIFY);
-
-    if (i_mm->t->len < INITIAL_SIZE) {
-        expand(i_mm, INITIAL_SIZE);
+static VALUE load_value(mm_ipc *i_mm, VALUE _offset) {
+    Check_Type(_offset, T_FIXNUM);
+    size_t offset = NUM2UINT(_offset);
+    if ((i_mm->t->real + sizeof(double)) <= offset) {
+        rb_raise(rb_eIndexError, "offset %zu out of string", offset);
     }
 
-    save_used(i_mm, NUM2UINT(value));
-    return value;
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    double value;
+    memcpy(&value, pos, sizeof(double));
+    return DBL2NUM(value);
 }
 
-VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
-    Check_Type(positions, T_HASH);
-    Check_Type(key, T_STRING);
+inline uint32_t load_used(mm_ipc *i_mm) {
+    uint32_t used = *((uint32_t *)i_mm->t->addr);
 
-    VALUE position = rb_hash_lookup(positions, key);
-    if (position != Qnil){
-        return position;
+    if (used == 0) {
+        used = START_POSITION;
     }
+    return used;
+}
 
-    mm_ipc *i_mm;
-    GET_MMAP(self, i_mm, MM_MODIFY);
+inline void save_used(mm_ipc *i_mm, uint32_t used) { *((uint32_t *)i_mm->t->addr) = used; }
 
+static VALUE initialize_entry(mm_ipc *i_mm, VALUE positions, VALUE key, VALUE value) {
     if (i_mm->t->flag & MM_FROZEN) {
         rb_error_frozen("mmap");
     }
@@ -139,25 +136,68 @@ VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
     while (i_mm->t->len < (used + entry_length)) {
         expand(i_mm, i_mm->t->len * 2);
     }
-
     save_entry(i_mm, used, key, value);
     save_used(i_mm, used + entry_length);
+
     return rb_hash_aset(positions, key, INT2NUM(used + value_offset));
 }
 
-VALUE method_get_double(VALUE self, VALUE index) {
+VALUE method_fetch_entry(VALUE self, VALUE positions, VALUE key, VALUE default_value) {
+    Check_Type(positions, T_HASH);
+    Check_Type(key, T_STRING);
+
+    mm_ipc *i_mm;
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    VALUE position = rb_hash_lookup(positions, key);
+
+    if (position != Qnil) {
+        return load_value(i_mm, position);
+    }
+
+    position = initialize_entry(i_mm, positions, key, default_value);
+    return load_value(i_mm, position);
+}
+
+VALUE method_upsert_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
+    Check_Type(positions, T_HASH);
+    Check_Type(key, T_STRING);
+
+    mm_ipc *i_mm;
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    VALUE position = rb_hash_lookup(positions, key);
+
+    if (position != Qnil) {
+        save_value(i_mm, position, value);
+        return load_value(i_mm, position);
+    }
+
+    position = initialize_entry(i_mm, positions, key, value);
+    return load_value(i_mm, position);
+}
+
+VALUE method_load_used(VALUE self) {
     mm_ipc *i_mm;
+
     GET_MMAP(self, i_mm, MM_MODIFY);
+    return UINT2NUM(load_used(i_mm));
+}
+
+VALUE method_save_used(VALUE self, VALUE value) {
+    Check_Type(value, T_FIXNUM);
+    mm_ipc *i_mm;
 
-    Check_Type(index, T_FIXNUM);
-    size_t idx = NUM2UINT(index);
+    GET_MMAP(self, i_mm, MM_MODIFY);
 
-    if ((i_mm->t->real + sizeof(double)) <= idx) {
-        rb_raise(rb_eIndexError, "index %zu out of string", idx);
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
     }
 
-    double tmp;
+    if (i_mm->t->len < INITIAL_SIZE) {
+        expand(i_mm, INITIAL_SIZE);
+    }
 
-    memcpy(&tmp, (char *)i_mm->t->addr + idx, sizeof(double));
-    return DBL2NUM(tmp);
-}
+    save_used(i_mm, NUM2UINT(value));
+    return value;
+}

data/ext/fast_mmaped_file/value_access.h

@@ -5,8 +5,10 @@ VALUE method_load_used(VALUE self);
 
 VALUE method_save_used(VALUE self, VALUE value);
 
-VALUE method_add_entry(VALUE self, VALUE positions, VALUE key, VALUE value);
-
 VALUE method_get_double(VALUE self, VALUE index);
 
+VALUE method_fetch_entry(VALUE self, VALUE positions, VALUE key, VALUE default_value);
+
+VALUE method_upsert_entry(VALUE self, VALUE positions, VALUE key, VALUE value);
+
 #endif

data/lib/prometheus/client/helper/entry_parser.rb

@@ -27,7 +27,7 @@ module Prometheus
         end
 
         def pid
-          parts[2..-1].join('_')
+          (parts[2..-1] || []).join('_')
         end
 
         def multiprocess_mode

data/lib/prometheus/client/helper/mmaped_file.rb

@@ -1,18 +1,16 @@
 require 'prometheus/client/helper/entry_parser'
 require 'prometheus/client/helper/file_locker'
 require 'fast_mmaped_file'
-require 'mmap'
 
 module Prometheus
   module Client
     module Helper
-      class MmapedFile < Mmap
+      class MmapedFile < FastMmapedFile
         include EntryParser
-        include FastMmapedFile
 
         attr_reader :filepath, :size
 
-        def initialize(filepath, mode = 'r', protection = Mmap::MAP_SHARED, options = {})
+        def initialize(filepath)
           @filepath = filepath
 
           File.open(filepath, 'a+b') do |file|
@@ -20,7 +18,7 @@ module Prometheus
             @size = file.size
           end
 
-          super(filepath, mode, protection, options)
+          super(filepath)
         end
 
         def close
@@ -37,7 +35,7 @@ module Prometheus
 
         class << self
           def open(filepath)
-            MmapedFile.new(filepath, 'rw', Mmap::MAP_SHARED)
+            MmapedFile.new(filepath)
           end
 
           def ensure_exclusive_file(file_prefix = 'mmaped_file')

data/lib/prometheus/client/helper/plain_file.rb

@@ -8,7 +8,7 @@ module Prometheus
         include EntryParser
         attr_reader :filepath
 
-        def data
+        def source
           @data ||= File.read(filepath, mode: 'rb')
         end
 
@@ -17,11 +17,11 @@ module Prometheus
         end
 
         def slice(*args)
-          data.slice(*args)
+          source.slice(*args)
         end
 
         def size
-          data.length
+          source.length
         end
       end
     end

data/lib/prometheus/client/mmaped_dict.rb

@@ -39,16 +39,11 @@ module Prometheus
       end
 
       def read_value(key)
-        pos = @m.add_entry(@positions, key, 0.0)
-
-        @m.get_double(pos)
+        @m.fetch_entry(@positions, key, 0.0)
       end
 
       def write_value(key, value)
-        pos = @m.add_entry(@positions, key, 0.0)
-
-        # pos = @positions[key]
-        @m[pos..pos + 7] = [value].pack('d')
+        @m.upsert_entry(@positions, key, value)
       end
 
       def path

data/lib/prometheus/client/simple_value.rb

@@ -1,5 +1,4 @@
 require 'json'
-require 'mmap'
 
 module Prometheus
   module Client

data/lib/prometheus/client/support/unicorn.rb

@@ -12,6 +12,15 @@ module Prometheus
         end
 
         def self.worker_id
+          match = $0.match(/worker\[([^\]]+)\]/)
+          if match
+            match[1]
+          else
+            object_based_worker_id
+          end
+        end
+
+        def self.object_based_worker_id
           return unless defined?(::Unicorn::Worker)
 
           workers = ObjectSpace.each_object(::Unicorn::Worker)

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '0.7.0.beta41'.freeze
+    VERSION = '0.7.0.beta42'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 0.7.0.beta41
+  version: 0.7.0.beta42
 platform: ruby
 authors:
 - Tobias Schmidt
@@ -9,28 +9,8 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-30 00:00:00.000000000 Z
+date: 2017-12-06 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: mmap2
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.2.9
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.2.9
 - !ruby/object:Gem::Dependency
   name: fuzzbert
   requirement: !ruby/object:Gem::Requirement
@@ -112,9 +92,8 @@ files:
 - ext/fast_mmaped_file/file_reading.h
 - ext/fast_mmaped_file/globals.h
 - ext/fast_mmaped_file/hashmap.c
-- ext/fast_mmaped_file/hashmap.h
 - ext/fast_mmaped_file/jsmn.c
-- ext/fast_mmaped_file/jsmn.h
+- ext/fast_mmaped_file/mmap.c
 - ext/fast_mmaped_file/mmap.h
 - ext/fast_mmaped_file/rendering.c
 - ext/fast_mmaped_file/rendering.h
@@ -150,7 +129,6 @@ files:
 - lib/prometheus/client/support/unicorn.rb
 - lib/prometheus/client/uses_value_type.rb
 - lib/prometheus/client/version.rb
-- lib/prometheus/client/version.rb.orig
 homepage: https://gitlab.com/gitlab-org/prometheus-client-mmap
 licenses:
 - Apache 2.0

data/ext/fast_mmaped_file/hashmap.h

@@ -1,266 +0,0 @@
-/*
- * Copyright (c) 2016-2017 David Leeds <davidesleeds@gmail.com>
- *
- * Hashmap is free software; you can redistribute it and/or modify
- * it under the terms of the MIT license. See LICENSE for details.
- */
-
-#ifndef __HASHMAP_H__
-#define __HASHMAP_H__
-
-#include <stddef.h>
-
-/*
- * Define HASHMAP_METRICS to compile in performance analysis
- * functions for use in assessing hash function performance.
- */
-/* #define HASHMAP_METRICS */
-
-/*
- * Define HASHMAP_NOASSERT to compile out all assertions used internally.
- */
-/* #define HASHMAP_NOASSERT */
-
-/*
- * Macros to declare type-specific versions of hashmap_*() functions to
- * allow compile-time type checking and avoid the need for type casting.
- */
-#define HASHMAP_FUNCS_DECLARE(name, key_type, data_type)		\
-	data_type *name##_hashmap_put(struct hashmap *map, key_type *key, \
-		data_type *data);					\
-	data_type *name##_hashmap_get(const struct hashmap *map,	\
-		key_type *key);						\
-	data_type *name##_hashmap_remove(struct hashmap *map,		\
-		key_type *key);						\
-	key_type *name##_hashmap_iter_get_key(				\
-		const struct hashmap_iter *iter);			\
-	data_type *name##_hashmap_iter_get_data(			\
-		const struct hashmap_iter *iter);			\
-	void name##_hashmap_iter_set_data(const struct hashmap_iter *iter, \
-		data_type *data);					\
-	int name##_hashmap_foreach(const struct hashmap *map,		\
-		int (*func)(key_type *, data_type *, void *), void *arg);
-
-#define HASHMAP_FUNCS_CREATE(name, key_type, data_type)			\
-	data_type *name##_hashmap_put(struct hashmap *map, key_type *key, \
-		data_type *data)					\
-	{								\
-		return (data_type *)hashmap_put(map, (const void *)key,	\
-		    (void *)data);					\
-	}								\
-	data_type *name##_hashmap_get(const struct hashmap *map,	\
-		key_type *key)						\
-	{								\
-		return (data_type *)hashmap_get(map, (const void *)key); \
-	}								\
-	data_type *name##_hashmap_remove(struct hashmap *map,		\
-		key_type *key)						\
-	{								\
-		return (data_type *)hashmap_remove(map, (const void *)key); \
-	}								\
-	key_type *name##_hashmap_iter_get_key(				\
-		const struct hashmap_iter *iter)			\
-	{								\
-		return (key_type *)hashmap_iter_get_key(iter);		\
-	}								\
-	data_type *name##_hashmap_iter_get_data(			\
-		const struct hashmap_iter *iter)			\
-	{								\
-		return (data_type *)hashmap_iter_get_data(iter);	\
-	}								\
-	void name##_hashmap_iter_set_data(const struct hashmap_iter *iter, \
-		data_type *data) 					\
-	{								\
-		hashmap_iter_set_data(iter, (void *)data);		\
-	}								\
-	struct __##name##_hashmap_foreach_state {			\
-		int (*func)(key_type *, data_type *, void *);		\
-		void *arg;						\
-	};								\
-	static inline int __##name##_hashmap_foreach_callback(const void *key, \
-		void *data, void *arg)					\
-	{								\
-		struct __##name##_hashmap_foreach_state *s =		\
-		    (struct __##name##_hashmap_foreach_state *)arg;	\
-		return s->func((key_type *)key, (data_type *)data, s->arg); \
-	}								\
-	int name##_hashmap_foreach(const struct hashmap *map,		\
-		int (*func)(key_type *, data_type *, void *), void *arg) \
-	{								\
-		struct __##name##_hashmap_foreach_state s = { func, arg }; \
-		return hashmap_foreach(map,				\
-		    __##name##_hashmap_foreach_callback, &s);		\
-	}
-
-
-struct hashmap_iter;
-struct hashmap_entry;
-
-/*
- * The hashmap state structure.
- */
-struct hashmap {
-	size_t table_size_init;
-	size_t table_size;
-	size_t num_entries;
-	struct hashmap_entry *table;
-	size_t (*hash)(const void *);
-	int (*key_compare)(const void *, const void *);
-	void *(*key_alloc)(const void *);
-	void (*key_free)(void *);
-};
-
-/*
- * Initialize an empty hashmap.  A hash function and a key comparator are
- * required.
- *
- * hash_func should return an even distribution of numbers between 0
- * and SIZE_MAX varying on the key provided.
- *
- * key_compare_func should return 0 if the keys match, and non-zero otherwise.
- *
- * initial_size is optional, and may be set to the max number of entries
- * expected to be put in the hash table.  This is used as a hint to
- * pre-allocate the hash table to the minimum size to avoid gratuitous rehashes.
- * If initial_size 0, a default size will be used.
- *
- * Returns 0 on success and -errno on failure.
- */
-int hashmap_init(struct hashmap *map, size_t (*hash_func)(const void *),
-	int (*key_compare_func)(const void *, const void *),
-	size_t initial_size);
-
-/*
- * Free the hashmap and all associated memory.
- */
-void hashmap_destroy(struct hashmap *map);
-
-/*
- * Enable internal memory allocation and management of hash keys.
- */
-void hashmap_set_key_alloc_funcs(struct hashmap *map,
-	void *(*key_alloc_func)(const void *),
-	void (*key_free_func)(void *));
-
-/*
- * Add an entry to the hashmap.  If an entry with a matching key already
- * exists and has a data pointer associated with it, the existing data
- * pointer is returned, instead of assigning the new value.  Compare
- * the return value with the data passed in to determine if a new entry was
- * created.  Returns NULL if memory allocation failed.
- */
-void *hashmap_put(struct hashmap *map, const void *key, void *data);
-
-/*
- * Return the data pointer, or NULL if no entry exists.
- */
-void *hashmap_get(const struct hashmap *map, const void *key);
-
-/*
- * Remove an entry with the specified key from the map.
- * Returns the data pointer, or NULL, if no entry was found.
- */
-void *hashmap_remove(struct hashmap *map, const void *key);
-
-/*
- * Remove all entries.
- */
-void hashmap_clear(struct hashmap *map);
-
-/*
- * Remove all entries and reset the hash table to its initial size.
- */
-void hashmap_reset(struct hashmap *map);
-
-/*
- * Return the number of entries in the hash map.
- */
-size_t hashmap_size(const struct hashmap *map);
-
-/*
- * Get a new hashmap iterator.  The iterator is an opaque
- * pointer that may be used with hashmap_iter_*() functions.
- * Hashmap iterators are INVALID after a put or remove operation is performed.
- * hashmap_iter_remove() allows safe removal during iteration.
- */
-struct hashmap_iter *hashmap_iter(const struct hashmap *map);
-
-/*
- * Return an iterator to the next hashmap entry.  Returns NULL if there are
- * no more entries.
- */
-struct hashmap_iter *hashmap_iter_next(const struct hashmap *map,
-	const struct hashmap_iter *iter);
-
-/*
- * Remove the hashmap entry pointed to by this iterator and returns an
- * iterator to the next entry.  Returns NULL if there are no more entries.
- */
-struct hashmap_iter *hashmap_iter_remove(struct hashmap *map,
-	const struct hashmap_iter *iter);
-
-/*
- * Return the key of the entry pointed to by the iterator.
- */
-const void *hashmap_iter_get_key(const struct hashmap_iter *iter);
-
-/*
- * Return the data of the entry pointed to by the iterator.
- */
-void *hashmap_iter_get_data(const struct hashmap_iter *iter);
-
-/*
- * Set the data pointer of the entry pointed to by the iterator.
- */
-void hashmap_iter_set_data(const struct hashmap_iter *iter, void *data);
-
-/*
- * Invoke func for each entry in the hashmap.  Unlike the hashmap_iter_*()
- * interface, this function supports calls to hashmap_remove() during iteration.
- * However, it is an error to put or remove an entry other than the current one,
- * and doing so will immediately halt iteration and return an error.
- * Iteration is stopped if func returns non-zero.  Returns func's return
- * value if it is < 0, otherwise, 0.
- */
-int hashmap_foreach(const struct hashmap *map,
-	int (*func)(const void *, void *, void *), void *arg);
-
-/*
- * Default hash function for string keys.
- * This is an implementation of the well-documented Jenkins one-at-a-time
- * hash function.
- */
-size_t hashmap_hash_string(const void *key);
-
-/*
- * Default key comparator function for string keys.
- */
-int hashmap_compare_string(const void *a, const void *b);
-
-/*
- * Default key allocation function for string keys.  Use free() for the
- * key_free_func.
- */
-void *hashmap_alloc_key_string(const void *key);
-
-
-#ifdef HASHMAP_METRICS
-/*
- * Return the load factor.
- */
-double hashmap_load_factor(const struct hashmap *map);
-
-/*
- * Return the average number of collisions per entry.
- */
-double hashmap_collisions_mean(const struct hashmap *map);
-
-/*
- * Return the variance between entry collisions.  The higher the variance,
- * the more likely the hash function is poor and is resulting in clustering.
- */
-double hashmap_collisions_variance(const struct hashmap *map);
-#endif
-
-
-#endif /* __HASHMAP_H__ */