prometheus-client-mmap 0.20.3-x86_64-linux

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a29c0bc9f71522519571c299d640a45952cadbb609671d4e47fdc299bed5dee3
+  data.tar.gz: 353185f7961a62ad1385086671616c87928180755a486630403272a71e792cea
+SHA512:
+  metadata.gz: 425e95cdf42d932c033b50370525bfc64a937619bbfa69fd6026eb17a379d4e9b7731586916d18bd25ea20ab7b3a7299f4eb5d5005cdfc7687d403396175cd6f
+  data.tar.gz: 48b52606c146743cc655bea58779ceb288b6730306f77760af7f9037d57bd5d700647aea7b2f507006d9dcc99025401579a78c05d6f4b82092cf854bf5913cbb

data/README.md

@@ -0,0 +1,253 @@
+# Prometheus Ruby Mmap Client
+
+This Prometheus library is fork of [Prometheus Ruby Client](https://github.com/prometheus/client_ruby)
+that uses mmap'ed files to share metrics from multiple processes.
+This allows efficient metrics processing for Ruby web apps running in multiprocess setups like Unicorn.
+
+A suite of instrumentation metric primitives for Ruby that can be exposed
+through a HTTP interface. Intended to be used together with a
+[Prometheus server][1].
+
+[![Gem Version][4]](http://badge.fury.io/rb/prometheus-client-mmap)
+[![Build Status][3]](https://gitlab.com/gitlab-org/prometheus-client-mmap/commits/master)
+[![Dependency Status][5]](https://gemnasium.com/prometheus/prometheus-client-mmap)
+
+## Usage
+
+### Overview
+
+```ruby
+require 'prometheus/client'
+
+# returns a default registry
+prometheus = Prometheus::Client.registry
+
+# create a new counter metric
+http_requests = Prometheus::Client::Counter.new(:http_requests, 'A counter of HTTP requests made')
+# register the metric
+prometheus.register(http_requests)
+
+# equivalent helper function
+http_requests = prometheus.counter(:http_requests, 'A counter of HTTP requests made')
+
+# start using the counter
+http_requests.increment
+```
+
+## Rust extension (experimental)
+
+In an effort to improve maintainability, there is now an optional Rust
+implementation that reads the metric files and outputs the multiprocess
+metrics to text. If `rustc` is available, then the Rust extension will
+be built automatically. The `use_rust` keyword argument can be used:
+
+```ruby
+puts Prometheus::Client::Formats::Text.marshal_multiprocess(use_rust: true)
+```
+
+Note that this parameter will likely be deprecated and removed once the Rust
+extension becomes the default mode.
+
+### Rack middleware
+
+There are two [Rack][2] middlewares available, one to expose a metrics HTTP
+endpoint to be scraped by a prometheus server ([Exporter][9]) and one to trace all HTTP
+requests ([Collector][10]).
+
+It's highly recommended to enable gzip compression for the metrics endpoint,
+for example by including the `Rack::Deflater` middleware.
+
+```ruby
+# config.ru
+
+require 'rack'
+require 'prometheus/client/rack/collector'
+require 'prometheus/client/rack/exporter'
+
+use Rack::Deflater, if: ->(env, status, headers, body) { body.any? && body[0].length > 512 }
+use Prometheus::Client::Rack::Collector
+use Prometheus::Client::Rack::Exporter
+
+run ->(env) { [200, {'Content-Type' => 'text/html'}, ['OK']] }
+```
+
+Start the server and have a look at the metrics endpoint:
+[http://localhost:5000/metrics](http://localhost:5000/metrics).
+
+For further instructions and other scripts to get started, have a look at the
+integrated [example application](examples/rack/README.md).
+
+### Pushgateway
+
+The Ruby client can also be used to push its collected metrics to a
+[Pushgateway][8]. This comes in handy with batch jobs or in other scenarios
+where it's not possible or feasible to let a Prometheus server scrape a Ruby
+process. TLS and HTTP basic authentication are supported.
+
+```ruby
+require 'prometheus/client'
+require 'prometheus/client/push'
+
+registry = Prometheus::Client.registry
+# ... register some metrics, set/increment/observe/etc. their values
+
+# push the registry state to the default gateway
+Prometheus::Client::Push.new(job: 'my-batch-job').add(registry)
+
+# optional: specify a grouping key that uniquely identifies a job instance, and gateway.
+#
+# Note: the labels you use in the grouping key must not conflict with labels set on the
+# metrics being pushed. If they do, an error will be raised.
+Prometheus::Client::Push.new(
+  job: 'my-batch-job',
+  gateway: 'https://example.domain:1234',
+  grouping_key: { instance: 'some-instance', extra_key: 'foobar' }
+).add(registry)
+
+# If you want to replace any previously pushed metrics for a given grouping key,
+# use the #replace method.
+#
+# Unlike #add, this will completely replace the metrics under the specified grouping key
+# (i.e. anything currently present in the pushgateway for the specified grouping key, but
+# not present in the registry for that grouping key will be removed).
+#
+# See https://github.com/prometheus/pushgateway#put-method for a full explanation.
+Prometheus::Client::Push.new(job: 'my-batch-job').replace(registry)
+
+# If you want to delete all previously pushed metrics for a given grouping key,
+# use the #delete method.
+Prometheus::Client::Push.new(job: 'my-batch-job').delete
+```
+
+## Metrics
+
+The following metric types are currently supported.
+
+### Counter
+
+Counter is a metric that exposes merely a sum or tally of things.
+
+```ruby
+counter = Prometheus::Client::Counter.new(:service_requests_total, '...')
+
+# increment the counter for a given label set
+counter.increment({ service: 'foo' })
+
+# increment by a given value
+counter.increment({ service: 'bar' }, 5)
+
+# get current value for a given label set
+counter.get({ service: 'bar' })
+# => 5
+```
+
+### Gauge
+
+Gauge is a metric that exposes merely an instantaneous value or some snapshot
+thereof.
+
+```ruby
+gauge = Prometheus::Client::Gauge.new(:room_temperature_celsius, '...')
+
+# set a value
+gauge.set({ room: 'kitchen' }, 21.534)
+
+# retrieve the current value for a given label set
+gauge.get({ room: 'kitchen' })
+# => 21.534
+```
+
+### Histogram
+
+A histogram samples observations (usually things like request durations or
+response sizes) and counts them in configurable buckets. It also provides a sum
+of all observed values.
+
+```ruby
+histogram = Prometheus::Client::Histogram.new(:service_latency_seconds, '...')
+
+# record a value
+histogram.observe({ service: 'users' }, Benchmark.realtime { service.call(arg) })
+
+# retrieve the current bucket values
+histogram.get({ service: 'users' })
+# => { 0.005 => 3, 0.01 => 15, 0.025 => 18, ..., 2.5 => 42, 5 => 42, 10 = >42 }
+```
+
+### Summary
+
+Summary, similar to histograms, is an accumulator for samples. It captures
+Numeric data and provides an efficient percentile calculation mechanism.
+
+```ruby
+summary = Prometheus::Client::Summary.new(:service_latency_seconds, '...')
+
+# record a value
+summary.observe({ service: 'database' }, Benchmark.realtime { service.call() })
+
+# retrieve the current quantile values
+summary.get({ service: 'database' })
+# => { 0.5 => 0.1233122, 0.9 => 3.4323, 0.99 => 5.3428231 }
+```
+
+## Configuration
+
+### Memory mapped files storage location
+
+Set `prometheus_multiproc_dir` environment variable to the path where you want metric files to be stored. Example:
+
+```
+prometheus_multiproc_dir=/tmp
+```
+
+## Pitfalls
+
+### PID cardinality
+
+In multiprocess setup e.g. running under Unicorn, having worker process restart often can
+lead to performance problems when proccesing metric files. By default each process using
+Prometheus metrics will create a set of files based on that process PID. With high worker
+churn this will lead to creation of thousands of files and in turn will cause very noticable
+slowdown when displaying metrics
+
+To reduce this problem, a surrogate process id can be used. Set of all such IDs needs
+have low cardinality, and each process id must be unique among all running process.
+
+For Unicorn a worker id/number can be used to greatly speedup the metrics rendering.
+
+To use it add this line to your `configure` block:
+
+```ruby
+  config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider)
+```
+
+## Tools
+
+###`bin/parse`
+
+This command can be used to parse metric files located on the filesystem just like a metric exporter would.
+It outputs either `json` formatted raw data or digested data in prometheus `text` format.
+
+#### Usage:
+
+```bash
+$ ./bin/parse -h
+Usage: parse [options] files...
+    -t, --to-prometheus-text         format output using Prometheus text formatter
+    -p, --profile                    enable profiling
+    -h, --help                       Show this message
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+[1]: https://github.com/prometheus/prometheus
+[2]: http://rack.github.io/
+[3]: https://gitlab.com/gitlab-org/prometheus-client-mmap/badges/master/pipeline.svg
+[4]: https://badge.fury.io/rb/prometheus-client.svg
+[8]: https://github.com/prometheus/pushgateway
+[9]: lib/prometheus/client/rack/exporter.rb
+[10]: lib/prometheus/client/rack/collector.rb

data/ext/fast_mmaped_file/extconf.rb

@@ -0,0 +1,30 @@
+require 'mkmf'
+require 'fileutils'
+
+$CFLAGS << ' -std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra'
+
+if enable_config('fail-on-warning')
+  $CFLAGS << ' -Werror'
+end
+
+if enable_config('debug')
+  $CFLAGS << ' -O0 -g'
+end
+
+if enable_config('address-sanitizer')
+  $CFLAGS << ' -O -fsanitize=address -fno-omit-frame-pointer -g'
+end
+
+CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
+
+cwd = File.expand_path(File.dirname(__FILE__))
+vendor_dir = File.join(cwd, '../../vendor/c')
+src_dir = File.join(cwd, '../../ext/fast_mmaped_file')
+
+src_files = %W[#{vendor_dir}/jsmn/jsmn.c #{vendor_dir}/hashmap/src/hashmap.c]
+FileUtils.cp(src_files, src_dir)
+
+$INCFLAGS << " -I#{vendor_dir}/jsmn -I#{vendor_dir}/hashmap/src"
+
+dir_config('fast_mmaped_file')
+create_makefile('fast_mmaped_file')

data/ext/fast_mmaped_file/fast_mmaped_file.c

@@ -0,0 +1,122 @@
+#include <errno.h>
+#include <hashmap.h>
+#include <jsmn.h>
+#include <ruby.h>
+#include <ruby/intern.h>
+#include <sys/mman.h>
+
+#include "file_parsing.h"
+#include "file_reading.h"
+#include "globals.h"
+#include "mmap.h"
+#include "rendering.h"
+#include "utils.h"
+#include "value_access.h"
+
+VALUE MMAPED_FILE = Qnil;
+
+ID sym_min;
+ID sym_max;
+ID sym_livesum;
+ID sym_gauge;
+ID sym_pid;
+ID sym_samples;
+
+VALUE prom_eParsingError;
+
+int aggregate_files(struct hashmap *map, VALUE list_of_files) {
+    buffer_t reading_buffer;
+    memset(&reading_buffer, 0, sizeof(buffer_t));
+
+    for (int i = 0; i < RARRAY_LEN(list_of_files); i++) {
+        VALUE params = RARRAY_PTR(list_of_files)[i];
+        file_t file;
+
+        if (!file_open_from_params(&file, params)) {
+            buffer_dispose(&reading_buffer);
+            return 0;
+        }
+
+        if (!read_from_file(&file, &reading_buffer)) {
+            buffer_dispose(&reading_buffer);
+            file_close(&file);
+            return 0;
+        }
+
+        if (!process_buffer(&file, &reading_buffer, map)) {
+            buffer_dispose(&reading_buffer);
+            file_close(&file);
+            return 0;
+        }
+
+        if (!file_close(&file)) {
+            buffer_dispose(&reading_buffer);
+            return 0;
+        }
+    }
+
+    buffer_dispose(&reading_buffer);
+    return 1;
+}
+
+VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list) {
+    struct hashmap map;
+    hashmap_setup(&map);
+
+    if (!aggregate_files(&map, file_list)) {  // all entries in map are now copies that need to be disposed
+        hashmap_destroy(&map);
+        raise_last_exception();
+        return Qnil;
+    }
+
+    entry_t **sorted_entries;
+
+    if (!sort_map_entries(&map, &sorted_entries)) {
+        hashmap_destroy(&map);
+
+        raise_last_exception();
+        return Qnil;
+    }
+
+    VALUE rv = rb_str_new("", 0);
+    if (!entries_to_string(rv, sorted_entries, hashmap_size(&map))) {
+        free(sorted_entries);
+        hashmap_destroy(&map);
+
+        raise_last_exception();
+        return Qnil;
+    }
+
+    RB_GC_GUARD(file_list);  // ensure file list is not GCed before this point
+    free(sorted_entries);
+    hashmap_destroy(&map);
+    return rv;
+}
+
+void Init_fast_mmaped_file() {
+    sym_gauge = rb_intern("gauge");
+    sym_min = rb_intern("min");
+    sym_max = rb_intern("max");
+    sym_livesum = rb_intern("livesum");
+    sym_pid = rb_intern("pid");
+    sym_samples = rb_intern("samples");
+
+    prom_eParsingError = rb_define_class("PrometheusParsingError", rb_eRuntimeError);
+
+    MMAPED_FILE = rb_define_class("FastMmapedFile", rb_cObject);
+    rb_define_const(MMAPED_FILE, "MAP_SHARED", INT2FIX(MAP_SHARED));
+
+    rb_define_singleton_method(MMAPED_FILE, "to_metrics", method_to_metrics, 1);
+
+    rb_define_alloc_func(MMAPED_FILE, mm_s_alloc);
+    rb_define_singleton_method(MMAPED_FILE, "new", mm_s_new, -1);
+    rb_define_method(MMAPED_FILE, "initialize", mm_init, 1);
+    rb_define_method(MMAPED_FILE, "slice", mm_aref_m, -1);
+    rb_define_method(MMAPED_FILE, "sync", mm_msync, -1);
+    rb_define_method(MMAPED_FILE, "munmap", mm_unmap, 0);
+
+    rb_define_method(MMAPED_FILE, "used", method_load_used, 0);
+    rb_define_method(MMAPED_FILE, "used=", method_save_used, 1);
+    rb_define_method(MMAPED_FILE, "fetch_entry", method_fetch_entry, 3);
+    rb_define_method(MMAPED_FILE, "upsert_entry", method_upsert_entry, 3);
+}

data/ext/fast_mmaped_file/file_format.c

@@ -0,0 +1,5 @@
+#include "file_format.h"
+
+inline uint32_t padding_length(uint32_t key_length) {
+    return 8 - (sizeof(uint32_t) + key_length) % 8;  // padding | 8 byte aligned
+}

data/ext/fast_mmaped_file/file_format.h

@@ -0,0 +1,11 @@
+#ifndef FILE_FORMAT_H
+#define FILE_FORMAT_H
+
+#include <stdint.h>
+
+#define START_POSITION 8
+#define INITIAL_SIZE (2 * sizeof(int32_t))
+
+uint32_t padding_length(uint32_t key_length);
+
+#endif

data/ext/fast_mmaped_file/file_parsing.c

@@ -0,0 +1,195 @@
+#include "file_parsing.h"
+
+#include <hashmap.h>
+#include <jsmn.h>
+
+#include "file_format.h"
+#include "globals.h"
+#include "utils.h"
+
+HASHMAP_FUNCS_CREATE(entry, const entry_t, entry_t)
+
+typedef int (*compare_fn)(const void *a, const void *b);
+
+static size_t hashmap_hash_entry(const entry_t *entry) { return hashmap_hash_string(entry->json); }
+
+static int hashmap_compare_entry(const entry_t *a, const entry_t *b) {
+    if (a->json_size != b->json_size) {
+        return -1;
+    }
+
+    if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)) {
+        return -1;
+    }
+
+    return strncmp(a->json, b->json, a->json_size);
+}
+
+static void entry_free(entry_t *entry) {
+    free(entry->json);
+    free(entry);
+}
+
+static inline double min(double a, double b) { return a < b ? a : b; }
+
+static inline double max(double a, double b) { return a > b ? a : b; }
+
+static void merge_entry(entry_t *found, const entry_t *entry) {
+    if (entry->type == sym_gauge) {
+        if (entry->multiprocess_mode == sym_min) {
+            found->value = min(found->value, entry->value);
+        } else if (entry->multiprocess_mode == sym_max) {
+            found->value = max(found->value, entry->value);
+        } else if (entry->multiprocess_mode == sym_livesum) {
+            found->value += entry->value;
+        } else {
+            found->value = entry->value;
+        }
+    } else {
+        found->value += entry->value;
+    }
+}
+
+void merge_or_store(struct hashmap *map, entry_t *entry) {
+    entry_t *found = entry_hashmap_get(map, entry);
+    if (found) {
+        merge_entry(found, entry);
+        entry_free(entry);
+    } else {
+        entry_hashmap_put(map, entry, entry);  // use the hashmap like hashset actually
+    }
+}
+
+entry_t *entry_new(buffer_t *source, uint32_t pos, uint32_t encoded_len, file_t *file_info) {
+    entry_t *entry = calloc(1, sizeof(entry_t));
+    if (entry == NULL) {
+        return NULL;
+    }
+
+    entry->json = malloc(encoded_len + 1);
+    if (entry->json == NULL) {
+        free(entry);
+        return NULL;
+    }
+
+    memcpy(entry->json, source->buffer + pos, encoded_len);
+    entry->json[encoded_len] = '\0';
+    entry->json_size = encoded_len;
+
+    entry->pid = file_info->pid;
+    entry->multiprocess_mode = file_info->multiprocess_mode;
+    entry->type = file_info->type;
+
+    char *value_ptr = source->buffer + pos + encoded_len + padding_length(encoded_len);
+    memcpy(&(entry->value), value_ptr, sizeof(double));
+
+    return entry;
+}
+
+static int add_parsed_name(entry_t *entry) {
+    jsmn_parser parser;
+    jsmn_init(&parser);
+
+    jsmntok_t tokens[2];
+    memset(&tokens, 0, sizeof(tokens));
+
+    jsmn_parse(&parser, entry->json, entry->json_size, tokens, 2);
+    jsmntok_t *name_token = &tokens[1];
+
+    if (name_token->start < name_token->end && name_token->start > 0) {
+        entry->name = entry->json + name_token->start;
+        entry->name_len = name_token->end - name_token->start;
+        return 1;
+    }
+    return 0;
+}
+
+static int entry_lexical_comparator(const entry_t **a, const entry_t **b) {
+    size_t size_a = (*a)->json_size;
+    size_t size_b = (*b)->json_size;
+    size_t min_length = size_a < size_b ? size_a : size_b;
+
+    return strncmp((*a)->json, (*b)->json, min_length);
+}
+
+void hashmap_setup(struct hashmap *map) {
+    hashmap_init(map, (size_t(*)(const void *))hashmap_hash_entry,
+                 (int (*)(const void *, const void *))hashmap_compare_entry, 1000);
+    hashmap_set_key_alloc_funcs(map, NULL, (void (*)(void *))entry_free);
+}
+
+int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map) {
+    if (source->size < START_POSITION) {
+        // nothing to read
+        return 1;
+    }
+    uint32_t used;
+    memcpy(&used, source->buffer, sizeof(uint32_t));
+
+    if (used > source->size) {
+        save_exception(prom_eParsingError, "source file %s corrupted, used %u > file size %u", file_info->path, used,
+                       source->size);
+        return 0;
+    }
+
+    uint32_t pos = START_POSITION;
+    while (pos + sizeof(uint32_t) < used) {
+        uint32_t encoded_len;
+        memcpy(&encoded_len, source->buffer + pos, sizeof(uint32_t));
+        pos += sizeof(uint32_t);
+
+        uint32_t value_offset = encoded_len + padding_length(encoded_len);
+
+        if (pos + value_offset + sizeof(double) > used) {
+            save_exception(prom_eParsingError, "source file %s corrupted, used %u < stored data length %u",
+                           file_info->path, used, pos + value_offset + sizeof(double));
+            return 0;
+        }
+
+        entry_t *entry = entry_new(source, pos, encoded_len, file_info);
+        if (entry == NULL) {
+            save_exception(rb_eNoMemError, "Failed creating metrics entry");
+            return 0;
+        }
+
+        merge_or_store(map, entry);
+
+        pos += value_offset + sizeof(double);
+    }
+    return 1;
+}
+
+int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries) {
+    size_t num = hashmap_size(map);
+
+    entry_t **list = calloc(num, sizeof(entry_t *));
+
+    if (list == NULL) {
+        save_exception(rb_eNoMemError, "Couldn't allocate for %zu memory", num * sizeof(entry_t *));
+        return 0;
+    }
+
+    size_t cnt = 0;
+    struct hashmap_iter *iter;
+    for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
+        entry_t *entry = (entry_t *)entry_hashmap_iter_get_key(iter);
+        if (add_parsed_name(entry)) {
+            list[cnt] = entry;
+            cnt++;
+        }
+    }
+    if (cnt != num) {
+        save_exception(rb_eRuntimeError, "Processed entries %zu != map entries %zu", cnt, num);
+        free(list);
+        return 0;
+    }
+
+    qsort(list, cnt, sizeof(entry_t *), (compare_fn)&entry_lexical_comparator);
+    *sorted_entries = list;
+    return 1;
+}
+
+int is_pid_significant(const entry_t *e) {
+    ID mp = e->multiprocess_mode;
+    return e->type == sym_gauge && !(mp == sym_min || mp == sym_max || mp == sym_livesum);
+}

data/ext/fast_mmaped_file/file_parsing.h

@@ -0,0 +1,27 @@
+#ifndef FILE_PARSING_H
+#define FILE_PARSING_H
+#include <file_reading.h>
+#include <hashmap.h>
+#include <ruby.h>
+
+typedef struct {
+    char *json;
+    size_t json_size;
+    char *name;
+    size_t name_len;
+
+    ID multiprocess_mode;
+    ID type;
+    VALUE pid;
+
+    double value;
+} entry_t;
+
+void hashmap_setup(struct hashmap *map);
+
+int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map);
+int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries);
+
+int is_pid_significant(const entry_t *e);
+
+#endif