prometheus-client-mmap 0.20.3-x86_64-linux

data/ext/fast_mmaped_file/rendering.c

@@ -0,0 +1,199 @@
+#include "rendering.h"
+
+#include <float.h>
+#include <jsmn.h>
+
+#include "file_parsing.h"
+#include "globals.h"
+#include "utils.h"
+
+#ifndef DBL_DECIMAL_DIG
+#define DBL_DECIMAL_DIG 17
+#endif
+
+#define LABELS_START_OFFSET 4
+
+static inline int is_valid(const jsmntok_t *token) { return token->start < token->end && token->start >= 0; }
+static inline int valid_not_null(const entry_t *entry, const jsmntok_t *token) {
+    static const char null_s[] = "null";
+
+    if (!is_valid(token)) {
+        return 0;
+    }
+
+    if (token->type != JSMN_PRIMITIVE) {
+        return 1;
+    }
+
+    size_t token_len = token->end - token->start;
+
+    if (token_len < sizeof(null_s) - 1) {
+        return 1;
+    }
+
+    return strncmp(null_s, entry->json + token->start, sizeof(null_s) - 1) != 0;
+}
+
+static inline int append_token(VALUE string, const entry_t *entry, const jsmntok_t *token) {
+    if (!is_valid(token)) {
+        save_exception(prom_eParsingError, "parsing failed: %s", entry->json);
+        return 0;
+    }
+
+    rb_str_cat(string, entry->json + token->start, token->end - token->start);
+    return 1;
+}
+
+static int append_labels(VALUE string, const entry_t *entry, const int label_count, const jsmntok_t *tokens) {
+    if (label_count <= 0) {
+        if (is_pid_significant(entry)) {
+            rb_str_cat(string, "{pid=\"", 6);
+            rb_str_append(string, entry->pid);
+            rb_str_cat(string, "\"}", 2);
+        }
+        return 1;
+    }
+
+    rb_str_cat(string, "{", 1);
+
+    for (int i = 0; i < label_count; i++) {
+        int key = LABELS_START_OFFSET + i;
+        int val = LABELS_START_OFFSET + label_count + 1 + i;
+
+        if (!append_token(string, entry, &tokens[key])) {
+            return 0;
+        }
+        rb_str_cat(string, "=", 1);
+
+        rb_str_cat(string, "\"", 1);
+        if (valid_not_null(entry, &tokens[val])) {
+            append_token(string, entry, &tokens[val]);
+        }
+        rb_str_cat(string, "\"", 1);
+
+        if (i < label_count - 1) {
+            rb_str_cat(string, ",", 1);
+        }
+    }
+
+    if (is_pid_significant(entry)) {
+        rb_str_cat(string, ",pid=\"", 6);
+        rb_str_append(string, entry->pid);
+        rb_str_cat(string, "\"", 1);
+    }
+
+    rb_str_cat(string, "}", 1);
+    return 1;
+}
+
+static int validate_token_count(const int token_count, const entry_t *entry) {
+    if (token_count < 0) {
+        save_exception(prom_eParsingError, "too many labels or malformed json: %s", entry->json);
+        return 0;
+    }
+
+    if (token_count < LABELS_START_OFFSET) {
+        save_exception(prom_eParsingError, "malformed json: %s", entry->json);
+        return 0;
+    }
+
+    if ((token_count - (LABELS_START_OFFSET + 1)) % 2 != 0) {
+        save_exception(prom_eParsingError, "mismatched number of labels: %s", entry->json);
+        return 0;
+    }
+
+    return 1;
+}
+
+static int append_entry(VALUE string, const entry_t *entry) {
+    jsmn_parser parser;
+    jsmn_init(&parser);
+
+    jsmntok_t tokens[200];
+    jsmntok_t *name_token = &tokens[2];
+
+    int token_count = jsmn_parse(&parser, entry->json, entry->json_size, tokens, sizeof(tokens) / sizeof(tokens[0]));
+    int label_count = (token_count - (LABELS_START_OFFSET + 1)) / 2;
+
+    //
+    // Example JSON "['metric', 'name',['label_a','label_b'],['value_a', 'value_b']]"
+    // will be  parsed into following token list:
+    //
+    // [ "'metric', 'name',['label_a','label_b'],['value_a', 'value_b']",
+    // "metric", "name",
+    // "['label_a','label_b']", "label_a", "label_b",
+    // "['value_a', 'value_b']", "value_a", "value_b" ]
+    //
+    // where 'metric' is the name of the metric, while 'name'
+    // is in summaries and histograms to store names of "submetrics" like:
+    // histogram_name_bucket or histogram_name_sum
+
+    if (!validate_token_count(token_count, entry)) {
+        return 0;
+    }
+
+    if (!append_token(string, entry, name_token)) {
+        return 0;
+    }
+
+    if (!append_labels(string, entry, label_count, tokens)) {
+        return 0;
+    }
+
+    char value[255];
+
+    // print value with highest possible precision so that we do not lose any data
+    int written = snprintf(value, sizeof(value), " %.*g\n", DBL_DECIMAL_DIG, entry->value);
+    rb_str_cat(string, value, written);
+
+    return 1;
+}
+
+static void append_entry_head(VALUE string, const entry_t *entry) {
+    static const char help_beg[] = "# HELP ";
+    static const char help_fin[] = " Multiprocess metric\n";
+
+    rb_str_cat(string, help_beg, sizeof(help_beg) - 1);
+    rb_str_cat(string, entry->name, entry->name_len);
+    rb_str_cat(string, help_fin, sizeof(help_fin) - 1);
+
+    static const char type_beg[] = "# TYPE ";
+
+    rb_str_cat(string, type_beg, sizeof(type_beg) - 1);
+    rb_str_cat(string, entry->name, entry->name_len);
+    rb_str_cat(string, " ", 1);
+    rb_str_cat2(string, rb_id2name(entry->type));
+    rb_str_cat(string, "\n", 1);
+}
+
+static inline int entry_name_equal(const entry_t *a, const entry_t *b) {
+    if (a == NULL || b == NULL) {
+        return a == b;
+    }
+
+    if (a->name_len != b->name_len) {
+        return 0;
+    }
+
+    return strncmp(a->name, b->name, a->name_len) == 0;
+}
+
+int entries_to_string(VALUE string, entry_t **sorted_entries, size_t entries_count) {
+    entry_t *previous = NULL;
+
+    for (size_t i = 0; i < entries_count; i++) {
+        entry_t *entry = sorted_entries[i];
+
+        // when entry->name changes write metric header
+        if (!entry_name_equal(previous, entry)) {
+            previous = entry;
+            append_entry_head(string, entry);
+        }
+
+        if (!append_entry(string, entry)) {
+            return 0;
+        }
+    }
+
+    return 1;
+}

data/ext/fast_mmaped_file/rendering.h

@@ -0,0 +1,8 @@
+#ifndef RENDERING_H
+#define RENDERING_H
+#include <file_parsing.h>
+#include <ruby.h>
+
+int entries_to_string(VALUE string, entry_t **sorted_entries, size_t entries_count);
+
+#endif

data/ext/fast_mmaped_file/utils.c

@@ -0,0 +1,56 @@
+#include "utils.h"
+
+#include <errno.h>
+
+static void rb_save_exception(VALUE exception, VALUE message) {
+    VALUE current_thread = rb_thread_current();
+
+    rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception"), exception);
+    rb_thread_local_aset(current_thread, rb_intern("prometheus_last_exception_message"), message);
+}
+/* @deprecated - use with_exception ignoring return value */
+void save_exception(VALUE exception, const char *fmt, ...) {
+    va_list args;
+
+    va_start(args, fmt);
+    VALUE message = rb_vsprintf(fmt, args);
+
+    rb_save_exception(exception, message);
+    va_end(args);
+}
+
+int with_exception(VALUE exception, const char *fmt, ...) {
+    va_list args;
+
+    va_start(args, fmt);
+    VALUE message = rb_vsprintf(fmt, args);
+
+    rb_save_exception(exception, message);
+    va_end(args);
+
+    return FAILURE;
+}
+
+int with_exception_errno(VALUE exception, const char *fmt, ...) {
+    va_list args;
+
+    va_start(args, fmt);
+    VALUE message = rb_vsprintf(fmt, args);
+    rb_str_catf(message, " (%s)", strerror(errno));
+
+    rb_save_exception(exception, message);
+    va_end(args);
+
+    return FAILURE;
+}
+
+NORETURN(void raise_last_exception()) {
+    VALUE current_thread = rb_thread_current();
+    VALUE exception = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception"));
+    VALUE message = rb_thread_local_aref(current_thread, rb_intern("prometheus_last_exception_message"));
+    if (exception != Qnil) {
+        rb_raise(exception, "%s", StringValueCStr(message));
+    } else {
+        rb_raise(rb_eRuntimeError, "no exception found in thread local");
+    }
+}

data/ext/fast_mmaped_file/utils.h

@@ -0,0 +1,22 @@
+#ifndef UNUSED_H
+#define UNUSED_H
+#include <ruby.h>
+
+#ifdef UNUSED
+#elif defined(__GNUC__)
+#define UNUSED(x) UNUSED_##x __attribute__((unused))
+#elif defined(__LCLINT__)
+#define UNUSED(x) /*@unused@*/ x
+#else
+#define UNUSED(x) x
+#endif
+
+#define SUCCESS 1
+#define FAILURE 0
+
+NORETURN(void raise_last_exception());
+void save_exception(VALUE exception, const char *fmt, ...);
+int with_exception(VALUE exception, const char *fmt, ...);
+int with_exception_errno(VALUE exception, const char *fmt, ...);
+
+#endif

data/ext/fast_mmaped_file/value_access.c

@@ -0,0 +1,242 @@
+#include "value_access.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <ruby/intern.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "file_format.h"
+#include "mmap.h"
+#include "utils.h"
+
+static void close_file(mm_ipc *i_mm) {
+    close(i_mm->t->fd);
+    i_mm->t->fd = -1;
+}
+
+static int open_and_extend_file(mm_ipc *i_mm, size_t len) {
+    if (i_mm->t->fd < 0) {
+        int fd;
+
+        if ((fd = open(i_mm->t->path, i_mm->t->smode)) == -1) {
+            return with_exception_errno(rb_eArgError, "%s: Can't open %s", __FILE__, i_mm->t->path);
+        }
+        i_mm->t->fd = fd;
+    }
+
+    if (lseek(i_mm->t->fd, len - 1, SEEK_SET) == -1) {
+        close_file(i_mm);
+        return with_exception_errno(rb_eIOError, "Can't lseek %zu", len - 1);
+    }
+
+    if (write(i_mm->t->fd, "\000", 1) != 1) {
+        close_file(i_mm);
+        return with_exception_errno(rb_eIOError, "Can't extend %s", i_mm->t->path);
+    }
+    i_mm->t->len = len;
+
+    return SUCCESS;
+}
+
+static int perform_munmap(mm_ipc *i_mm) {
+    if (i_mm->t->addr != NULL && munmap(i_mm->t->addr, i_mm->t->len)) {
+        i_mm->t->addr = NULL;
+        return with_exception_errno(rb_eArgError, "munmap failed");
+    }
+
+    i_mm->t->addr = NULL;
+    i_mm->t->len = 0;
+    i_mm->t->real = 0;
+
+    return SUCCESS;
+}
+
+static int perform_mmap(mm_ipc *i_mm, size_t len) {
+    MMAP_RETTYPE addr = mmap(0, len, i_mm->t->pmode, i_mm->t->vscope, i_mm->t->fd, i_mm->t->offset);
+
+    if (addr == MAP_FAILED) {
+        return with_exception_errno(rb_eArgError, "mmap failed");
+    }
+
+    i_mm->t->addr = addr;
+    i_mm->t->len = len;
+    i_mm->t->real = len;
+
+    return SUCCESS;
+}
+
+static int expand(VALUE self, mm_ipc *i_mm, size_t len) {
+    if (len < i_mm->t->len) {
+        return with_exception(rb_eArgError, "Can't reduce the size of mmap");
+    }
+
+    if (!perform_munmap(i_mm)) {
+        return FAILURE;
+    }
+
+    if (!open_and_extend_file(i_mm, len)) {
+        return FAILURE;
+    }
+
+    if (!perform_mmap(i_mm, len)) {
+        close_file(i_mm);
+        return FAILURE;
+    }
+
+    if ((i_mm->t->flag & MM_LOCK) && mlock(i_mm->t->addr, len) == -1) {
+        return with_exception_errno(rb_eArgError, "mlock(%d)", errno);
+    }
+
+    mm_update(self);
+
+    return SUCCESS;
+}
+
+static void save_entry(mm_ipc *i_mm, size_t offset, VALUE key, VALUE value) {
+    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
+
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    memcpy(pos, &key_length, sizeof(uint32_t));
+    pos += sizeof(uint32_t);
+
+    memmove(pos, StringValuePtr(key), key_length);
+    pos += key_length;
+
+    memset(pos, ' ', padding_length(key_length));  // TODO: considder padding with /0
+    pos += padding_length(key_length);
+
+    double val = NUM2DBL(value);
+    memcpy(pos, &val, sizeof(double));
+}
+
+static void save_value(mm_ipc *i_mm, VALUE _offset, VALUE value) {
+    Check_Type(_offset, T_FIXNUM);
+    size_t offset = NUM2UINT(_offset);
+    if ((i_mm->t->real + sizeof(double)) <= offset) {
+        rb_raise(rb_eIndexError, "offset %zu out of string", offset);
+    }
+
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
+    }
+
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    double val = NUM2DBL(value);
+    memcpy(pos, &val, sizeof(double));
+}
+
+static VALUE load_value(mm_ipc *i_mm, VALUE _offset) {
+    Check_Type(_offset, T_FIXNUM);
+    size_t offset = NUM2UINT(_offset);
+    if ((i_mm->t->real + sizeof(double)) <= offset) {
+        rb_raise(rb_eIndexError, "offset %zu out of string", offset);
+    }
+
+    char *pos = (char *)i_mm->t->addr + offset;
+
+    double value;
+    memcpy(&value, pos, sizeof(double));
+    return DBL2NUM(value);
+}
+
+uint32_t load_used(mm_ipc *i_mm) {
+    uint32_t used = *((uint32_t *)i_mm->t->addr);
+
+    if (used == 0) {
+        used = START_POSITION;
+    }
+    return used;
+}
+
+void save_used(mm_ipc *i_mm, uint32_t used) { *((uint32_t *)i_mm->t->addr) = used; }
+
+static VALUE initialize_entry(VALUE self, mm_ipc *i_mm, VALUE positions, VALUE key, VALUE value) {
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
+    }
+
+    if (RSTRING_LEN(key) > INT32_MAX) {
+        rb_raise(rb_eArgError, "string length gt %d", INT32_MAX);
+    }
+
+    uint32_t key_length = (uint32_t)RSTRING_LEN(key);
+    uint32_t value_offset = sizeof(uint32_t) + key_length + padding_length(key_length);
+    uint32_t entry_length = value_offset + sizeof(double);
+
+    uint32_t used = load_used(i_mm);
+    while (i_mm->t->len < (used + entry_length)) {
+        if (!expand(self, i_mm, i_mm->t->len * 2)) {
+            raise_last_exception();
+        }
+    }
+
+    save_entry(i_mm, used, key, value);
+    save_used(i_mm, used + entry_length);
+
+    return rb_hash_aset(positions, key, INT2NUM(used + value_offset));
+}
+
+VALUE method_fetch_entry(VALUE self, VALUE positions, VALUE key, VALUE default_value) {
+    Check_Type(positions, T_HASH);
+    Check_Type(key, T_STRING);
+
+    mm_ipc *i_mm;
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    VALUE position = rb_hash_lookup(positions, key);
+
+    if (position != Qnil) {
+        return load_value(i_mm, position);
+    }
+
+    position = initialize_entry(self, i_mm, positions, key, default_value);
+    return load_value(i_mm, position);
+}
+
+VALUE method_upsert_entry(VALUE self, VALUE positions, VALUE key, VALUE value) {
+    Check_Type(positions, T_HASH);
+    Check_Type(key, T_STRING);
+
+    mm_ipc *i_mm;
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    VALUE position = rb_hash_lookup(positions, key);
+
+    if (position != Qnil) {
+        save_value(i_mm, position, value);
+        return load_value(i_mm, position);
+    }
+
+    position = initialize_entry(self, i_mm, positions, key, value);
+    return load_value(i_mm, position);
+}
+
+VALUE method_load_used(VALUE self) {
+    mm_ipc *i_mm;
+
+    GET_MMAP(self, i_mm, MM_MODIFY);
+    return UINT2NUM(load_used(i_mm));
+}
+
+VALUE method_save_used(VALUE self, VALUE value) {
+    Check_Type(value, T_FIXNUM);
+    mm_ipc *i_mm;
+
+    GET_MMAP(self, i_mm, MM_MODIFY);
+
+    if (i_mm->t->flag & MM_FROZEN) {
+        rb_error_frozen("mmap");
+    }
+
+    if (i_mm->t->len < INITIAL_SIZE) {
+        if (!expand(self, i_mm, INITIAL_SIZE)) {
+            raise_last_exception();
+        }
+    }
+
+    save_used(i_mm, NUM2UINT(value));
+    return value;
+}

data/ext/fast_mmaped_file/value_access.h

@@ -0,0 +1,15 @@
+#ifndef VALUE_ACCESS_H
+#define VALUE_ACCESS_H
+#include <ruby.h>
+
+VALUE method_load_used(VALUE self);
+
+VALUE method_save_used(VALUE self, VALUE value);
+
+VALUE method_get_double(VALUE self, VALUE index);
+
+VALUE method_fetch_entry(VALUE self, VALUE positions, VALUE key, VALUE default_value);
+
+VALUE method_upsert_entry(VALUE self, VALUE positions, VALUE key, VALUE value);
+
+#endif

data/ext/fast_mmaped_file_rs/.cargo/config.toml

@@ -0,0 +1,23 @@
+[target.aarch64-apple-darwin]
+# Without this flag, when linking static libruby, the linker removes symbols
+# (such as `_rb_ext_ractor_safe`) which it thinks are dead code... but they are
+# not, and they need to be included for the `embed` feature to work with static
+# Ruby.
+rustflags = [
+	"-C", "link-dead-code=on",
+	"-C", "link-arg=-undefined",
+	"-C", "link-arg=dynamic_lookup",
+]
+
+[target.x86_64-apple-darwin]
+rustflags = [
+	"-C", "link-dead-code=on",
+	"-C", "link-arg=-undefined",
+	"-C", "link-arg=dynamic_lookup",
+]
+
+[target.aarch64-unknown-linux-gnu]
+rustflags = [ "-C", "link-dead-code=on" ]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = [ "-C", "link-dead-code=on" ]