prometheus-client-mmap 0.20.3-aarch64-linux

Sign up to get free protection for your applications and to get access to all the features.
Files changed (92) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +253 -0
  3. data/ext/fast_mmaped_file/extconf.rb +30 -0
  4. data/ext/fast_mmaped_file/fast_mmaped_file.c +122 -0
  5. data/ext/fast_mmaped_file/file_format.c +5 -0
  6. data/ext/fast_mmaped_file/file_format.h +11 -0
  7. data/ext/fast_mmaped_file/file_parsing.c +195 -0
  8. data/ext/fast_mmaped_file/file_parsing.h +27 -0
  9. data/ext/fast_mmaped_file/file_reading.c +102 -0
  10. data/ext/fast_mmaped_file/file_reading.h +30 -0
  11. data/ext/fast_mmaped_file/globals.h +14 -0
  12. data/ext/fast_mmaped_file/mmap.c +427 -0
  13. data/ext/fast_mmaped_file/mmap.h +61 -0
  14. data/ext/fast_mmaped_file/rendering.c +199 -0
  15. data/ext/fast_mmaped_file/rendering.h +8 -0
  16. data/ext/fast_mmaped_file/utils.c +56 -0
  17. data/ext/fast_mmaped_file/utils.h +22 -0
  18. data/ext/fast_mmaped_file/value_access.c +242 -0
  19. data/ext/fast_mmaped_file/value_access.h +15 -0
  20. data/ext/fast_mmaped_file_rs/.cargo/config.toml +23 -0
  21. data/ext/fast_mmaped_file_rs/Cargo.lock +790 -0
  22. data/ext/fast_mmaped_file_rs/Cargo.toml +30 -0
  23. data/ext/fast_mmaped_file_rs/README.md +52 -0
  24. data/ext/fast_mmaped_file_rs/extconf.rb +30 -0
  25. data/ext/fast_mmaped_file_rs/src/error.rs +174 -0
  26. data/ext/fast_mmaped_file_rs/src/file_entry.rs +579 -0
  27. data/ext/fast_mmaped_file_rs/src/file_info.rs +190 -0
  28. data/ext/fast_mmaped_file_rs/src/lib.rs +79 -0
  29. data/ext/fast_mmaped_file_rs/src/macros.rs +14 -0
  30. data/ext/fast_mmaped_file_rs/src/map.rs +492 -0
  31. data/ext/fast_mmaped_file_rs/src/mmap.rs +151 -0
  32. data/ext/fast_mmaped_file_rs/src/parser.rs +346 -0
  33. data/ext/fast_mmaped_file_rs/src/raw_entry.rs +473 -0
  34. data/ext/fast_mmaped_file_rs/src/testhelper.rs +222 -0
  35. data/ext/fast_mmaped_file_rs/src/util.rs +121 -0
  36. data/lib/2.7/fast_mmaped_file.so +0 -0
  37. data/lib/2.7/fast_mmaped_file_rs.so +0 -0
  38. data/lib/3.0/fast_mmaped_file.so +0 -0
  39. data/lib/3.0/fast_mmaped_file_rs.so +0 -0
  40. data/lib/3.1/fast_mmaped_file.so +0 -0
  41. data/lib/3.1/fast_mmaped_file_rs.so +0 -0
  42. data/lib/3.2/fast_mmaped_file.so +0 -0
  43. data/lib/3.2/fast_mmaped_file_rs.so +0 -0
  44. data/lib/prometheus/client/configuration.rb +23 -0
  45. data/lib/prometheus/client/counter.rb +27 -0
  46. data/lib/prometheus/client/formats/text.rb +118 -0
  47. data/lib/prometheus/client/gauge.rb +40 -0
  48. data/lib/prometheus/client/helper/entry_parser.rb +132 -0
  49. data/lib/prometheus/client/helper/file_locker.rb +50 -0
  50. data/lib/prometheus/client/helper/json_parser.rb +23 -0
  51. data/lib/prometheus/client/helper/metrics_processing.rb +45 -0
  52. data/lib/prometheus/client/helper/metrics_representation.rb +51 -0
  53. data/lib/prometheus/client/helper/mmaped_file.rb +64 -0
  54. data/lib/prometheus/client/helper/plain_file.rb +29 -0
  55. data/lib/prometheus/client/histogram.rb +80 -0
  56. data/lib/prometheus/client/label_set_validator.rb +86 -0
  57. data/lib/prometheus/client/metric.rb +80 -0
  58. data/lib/prometheus/client/mmaped_dict.rb +79 -0
  59. data/lib/prometheus/client/mmaped_value.rb +154 -0
  60. data/lib/prometheus/client/page_size.rb +17 -0
  61. data/lib/prometheus/client/push.rb +203 -0
  62. data/lib/prometheus/client/rack/collector.rb +88 -0
  63. data/lib/prometheus/client/rack/exporter.rb +96 -0
  64. data/lib/prometheus/client/registry.rb +65 -0
  65. data/lib/prometheus/client/simple_value.rb +31 -0
  66. data/lib/prometheus/client/summary.rb +69 -0
  67. data/lib/prometheus/client/support/unicorn.rb +35 -0
  68. data/lib/prometheus/client/uses_value_type.rb +20 -0
  69. data/lib/prometheus/client/version.rb +5 -0
  70. data/lib/prometheus/client.rb +58 -0
  71. data/lib/prometheus.rb +3 -0
  72. data/vendor/c/hashmap/.gitignore +52 -0
  73. data/vendor/c/hashmap/LICENSE +21 -0
  74. data/vendor/c/hashmap/README.md +90 -0
  75. data/vendor/c/hashmap/_config.yml +1 -0
  76. data/vendor/c/hashmap/src/hashmap.c +692 -0
  77. data/vendor/c/hashmap/src/hashmap.h +267 -0
  78. data/vendor/c/hashmap/test/Makefile +22 -0
  79. data/vendor/c/hashmap/test/hashmap_test.c +608 -0
  80. data/vendor/c/jsmn/.travis.yml +4 -0
  81. data/vendor/c/jsmn/LICENSE +20 -0
  82. data/vendor/c/jsmn/Makefile +41 -0
  83. data/vendor/c/jsmn/README.md +168 -0
  84. data/vendor/c/jsmn/example/jsondump.c +126 -0
  85. data/vendor/c/jsmn/example/simple.c +76 -0
  86. data/vendor/c/jsmn/jsmn.c +314 -0
  87. data/vendor/c/jsmn/jsmn.h +76 -0
  88. data/vendor/c/jsmn/library.json +16 -0
  89. data/vendor/c/jsmn/test/test.h +27 -0
  90. data/vendor/c/jsmn/test/tests.c +407 -0
  91. data/vendor/c/jsmn/test/testutil.h +94 -0
  92. metadata +243 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: d05a8288379dd1dd067a6480768e1826140bffd66eb05a0d37a8a7ba5e7a1974
4
+ data.tar.gz: 718c25e9a40b9293083200e2f9916beb8bc99e222ddc4b50f1c0e9d924806e04
5
+ SHA512:
6
+ metadata.gz: 5220aa79b95c4112e13a43ac99dec26c52a11a037d40001f7a8432b63e1455bd9660356c89eac719d4a0aebbc7ccdfb9084ac24e885634e5944e16afc144c8f6
7
+ data.tar.gz: 527841138c4c42631a26f79e5ffe66abdad7f680d48ae6742f359de4ed29ff791de7649ceaaad32975bd5d075a58e8ae12232228b0db4efc693a0da3198c9d43
data/README.md ADDED
@@ -0,0 +1,253 @@
1
+ # Prometheus Ruby Mmap Client
2
+
3
+ This Prometheus library is fork of [Prometheus Ruby Client](https://github.com/prometheus/client_ruby)
4
+ that uses mmap'ed files to share metrics from multiple processes.
5
+ This allows efficient metrics processing for Ruby web apps running in multiprocess setups like Unicorn.
6
+
7
+ A suite of instrumentation metric primitives for Ruby that can be exposed
8
+ through a HTTP interface. Intended to be used together with a
9
+ [Prometheus server][1].
10
+
11
+ [![Gem Version][4]](http://badge.fury.io/rb/prometheus-client-mmap)
12
+ [![Build Status][3]](https://gitlab.com/gitlab-org/prometheus-client-mmap/commits/master)
13
+ [![Dependency Status][5]](https://gemnasium.com/prometheus/prometheus-client-mmap)
14
+
15
+ ## Usage
16
+
17
+ ### Overview
18
+
19
+ ```ruby
20
+ require 'prometheus/client'
21
+
22
+ # returns a default registry
23
+ prometheus = Prometheus::Client.registry
24
+
25
+ # create a new counter metric
26
+ http_requests = Prometheus::Client::Counter.new(:http_requests, 'A counter of HTTP requests made')
27
+ # register the metric
28
+ prometheus.register(http_requests)
29
+
30
+ # equivalent helper function
31
+ http_requests = prometheus.counter(:http_requests, 'A counter of HTTP requests made')
32
+
33
+ # start using the counter
34
+ http_requests.increment
35
+ ```
36
+
37
+ ## Rust extension (experimental)
38
+
39
+ In an effort to improve maintainability, there is now an optional Rust
40
+ implementation that reads the metric files and outputs the multiprocess
41
+ metrics to text. If `rustc` is available, then the Rust extension will
42
+ be built automatically. The `use_rust` keyword argument can be used:
43
+
44
+ ```ruby
45
+ puts Prometheus::Client::Formats::Text.marshal_multiprocess(use_rust: true)
46
+ ```
47
+
48
+ Note that this parameter will likely be deprecated and removed once the Rust
49
+ extension becomes the default mode.
50
+
51
+ ### Rack middleware
52
+
53
+ There are two [Rack][2] middlewares available, one to expose a metrics HTTP
54
+ endpoint to be scraped by a prometheus server ([Exporter][9]) and one to trace all HTTP
55
+ requests ([Collector][10]).
56
+
57
+ It's highly recommended to enable gzip compression for the metrics endpoint,
58
+ for example by including the `Rack::Deflater` middleware.
59
+
60
+ ```ruby
61
+ # config.ru
62
+
63
+ require 'rack'
64
+ require 'prometheus/client/rack/collector'
65
+ require 'prometheus/client/rack/exporter'
66
+
67
+ use Rack::Deflater, if: ->(env, status, headers, body) { body.any? && body[0].length > 512 }
68
+ use Prometheus::Client::Rack::Collector
69
+ use Prometheus::Client::Rack::Exporter
70
+
71
+ run ->(env) { [200, {'Content-Type' => 'text/html'}, ['OK']] }
72
+ ```
73
+
74
+ Start the server and have a look at the metrics endpoint:
75
+ [http://localhost:5000/metrics](http://localhost:5000/metrics).
76
+
77
+ For further instructions and other scripts to get started, have a look at the
78
+ integrated [example application](examples/rack/README.md).
79
+
80
+ ### Pushgateway
81
+
82
+ The Ruby client can also be used to push its collected metrics to a
83
+ [Pushgateway][8]. This comes in handy with batch jobs or in other scenarios
84
+ where it's not possible or feasible to let a Prometheus server scrape a Ruby
85
+ process. TLS and HTTP basic authentication are supported.
86
+
87
+ ```ruby
88
+ require 'prometheus/client'
89
+ require 'prometheus/client/push'
90
+
91
+ registry = Prometheus::Client.registry
92
+ # ... register some metrics, set/increment/observe/etc. their values
93
+
94
+ # push the registry state to the default gateway
95
+ Prometheus::Client::Push.new(job: 'my-batch-job').add(registry)
96
+
97
+ # optional: specify a grouping key that uniquely identifies a job instance, and gateway.
98
+ #
99
+ # Note: the labels you use in the grouping key must not conflict with labels set on the
100
+ # metrics being pushed. If they do, an error will be raised.
101
+ Prometheus::Client::Push.new(
102
+ job: 'my-batch-job',
103
+ gateway: 'https://example.domain:1234',
104
+ grouping_key: { instance: 'some-instance', extra_key: 'foobar' }
105
+ ).add(registry)
106
+
107
+ # If you want to replace any previously pushed metrics for a given grouping key,
108
+ # use the #replace method.
109
+ #
110
+ # Unlike #add, this will completely replace the metrics under the specified grouping key
111
+ # (i.e. anything currently present in the pushgateway for the specified grouping key, but
112
+ # not present in the registry for that grouping key will be removed).
113
+ #
114
+ # See https://github.com/prometheus/pushgateway#put-method for a full explanation.
115
+ Prometheus::Client::Push.new(job: 'my-batch-job').replace(registry)
116
+
117
+ # If you want to delete all previously pushed metrics for a given grouping key,
118
+ # use the #delete method.
119
+ Prometheus::Client::Push.new(job: 'my-batch-job').delete
120
+ ```
121
+
122
+ ## Metrics
123
+
124
+ The following metric types are currently supported.
125
+
126
+ ### Counter
127
+
128
+ Counter is a metric that exposes merely a sum or tally of things.
129
+
130
+ ```ruby
131
+ counter = Prometheus::Client::Counter.new(:service_requests_total, '...')
132
+
133
+ # increment the counter for a given label set
134
+ counter.increment({ service: 'foo' })
135
+
136
+ # increment by a given value
137
+ counter.increment({ service: 'bar' }, 5)
138
+
139
+ # get current value for a given label set
140
+ counter.get({ service: 'bar' })
141
+ # => 5
142
+ ```
143
+
144
+ ### Gauge
145
+
146
+ Gauge is a metric that exposes merely an instantaneous value or some snapshot
147
+ thereof.
148
+
149
+ ```ruby
150
+ gauge = Prometheus::Client::Gauge.new(:room_temperature_celsius, '...')
151
+
152
+ # set a value
153
+ gauge.set({ room: 'kitchen' }, 21.534)
154
+
155
+ # retrieve the current value for a given label set
156
+ gauge.get({ room: 'kitchen' })
157
+ # => 21.534
158
+ ```
159
+
160
+ ### Histogram
161
+
162
+ A histogram samples observations (usually things like request durations or
163
+ response sizes) and counts them in configurable buckets. It also provides a sum
164
+ of all observed values.
165
+
166
+ ```ruby
167
+ histogram = Prometheus::Client::Histogram.new(:service_latency_seconds, '...')
168
+
169
+ # record a value
170
+ histogram.observe({ service: 'users' }, Benchmark.realtime { service.call(arg) })
171
+
172
+ # retrieve the current bucket values
173
+ histogram.get({ service: 'users' })
174
+ # => { 0.005 => 3, 0.01 => 15, 0.025 => 18, ..., 2.5 => 42, 5 => 42, 10 = >42 }
175
+ ```
176
+
177
+ ### Summary
178
+
179
+ Summary, similar to histograms, is an accumulator for samples. It captures
180
+ Numeric data and provides an efficient percentile calculation mechanism.
181
+
182
+ ```ruby
183
+ summary = Prometheus::Client::Summary.new(:service_latency_seconds, '...')
184
+
185
+ # record a value
186
+ summary.observe({ service: 'database' }, Benchmark.realtime { service.call() })
187
+
188
+ # retrieve the current quantile values
189
+ summary.get({ service: 'database' })
190
+ # => { 0.5 => 0.1233122, 0.9 => 3.4323, 0.99 => 5.3428231 }
191
+ ```
192
+
193
+ ## Configuration
194
+
195
+ ### Memory mapped files storage location
196
+
197
+ Set `prometheus_multiproc_dir` environment variable to the path where you want metric files to be stored. Example:
198
+
199
+ ```
200
+ prometheus_multiproc_dir=/tmp
201
+ ```
202
+
203
+ ## Pitfalls
204
+
205
+ ### PID cardinality
206
+
207
+ In multiprocess setup e.g. running under Unicorn, having worker process restart often can
208
+ lead to performance problems when proccesing metric files. By default each process using
209
+ Prometheus metrics will create a set of files based on that process PID. With high worker
210
+ churn this will lead to creation of thousands of files and in turn will cause very noticable
211
+ slowdown when displaying metrics
212
+
213
+ To reduce this problem, a surrogate process id can be used. Set of all such IDs needs
214
+ have low cardinality, and each process id must be unique among all running process.
215
+
216
+ For Unicorn a worker id/number can be used to greatly speedup the metrics rendering.
217
+
218
+ To use it add this line to your `configure` block:
219
+
220
+ ```ruby
221
+ config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider)
222
+ ```
223
+
224
+ ## Tools
225
+
226
+ ###`bin/parse`
227
+
228
+ This command can be used to parse metric files located on the filesystem just like a metric exporter would.
229
+ It outputs either `json` formatted raw data or digested data in prometheus `text` format.
230
+
231
+ #### Usage:
232
+
233
+ ```bash
234
+ $ ./bin/parse -h
235
+ Usage: parse [options] files...
236
+ -t, --to-prometheus-text format output using Prometheus text formatter
237
+ -p, --profile enable profiling
238
+ -h, --help Show this message
239
+ ```
240
+
241
+ ## Development
242
+
243
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
244
+
245
+ To install this gem onto your local machine, run `bundle exec rake install`.
246
+
247
+ [1]: https://github.com/prometheus/prometheus
248
+ [2]: http://rack.github.io/
249
+ [3]: https://gitlab.com/gitlab-org/prometheus-client-mmap/badges/master/pipeline.svg
250
+ [4]: https://badge.fury.io/rb/prometheus-client.svg
251
+ [8]: https://github.com/prometheus/pushgateway
252
+ [9]: lib/prometheus/client/rack/exporter.rb
253
+ [10]: lib/prometheus/client/rack/collector.rb
@@ -0,0 +1,30 @@
1
+ require 'mkmf'
2
+ require 'fileutils'
3
+
4
+ $CFLAGS << ' -std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra'
5
+
6
+ if enable_config('fail-on-warning')
7
+ $CFLAGS << ' -Werror'
8
+ end
9
+
10
+ if enable_config('debug')
11
+ $CFLAGS << ' -O0 -g'
12
+ end
13
+
14
+ if enable_config('address-sanitizer')
15
+ $CFLAGS << ' -O -fsanitize=address -fno-omit-frame-pointer -g'
16
+ end
17
+
18
+ CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
19
+
20
+ cwd = File.expand_path(File.dirname(__FILE__))
21
+ vendor_dir = File.join(cwd, '../../vendor/c')
22
+ src_dir = File.join(cwd, '../../ext/fast_mmaped_file')
23
+
24
+ src_files = %W[#{vendor_dir}/jsmn/jsmn.c #{vendor_dir}/hashmap/src/hashmap.c]
25
+ FileUtils.cp(src_files, src_dir)
26
+
27
+ $INCFLAGS << " -I#{vendor_dir}/jsmn -I#{vendor_dir}/hashmap/src"
28
+
29
+ dir_config('fast_mmaped_file')
30
+ create_makefile('fast_mmaped_file')
@@ -0,0 +1,122 @@
1
+ #include <errno.h>
2
+ #include <hashmap.h>
3
+ #include <jsmn.h>
4
+ #include <ruby.h>
5
+ #include <ruby/intern.h>
6
+ #include <sys/mman.h>
7
+
8
+ #include "file_parsing.h"
9
+ #include "file_reading.h"
10
+ #include "globals.h"
11
+ #include "mmap.h"
12
+ #include "rendering.h"
13
+ #include "utils.h"
14
+ #include "value_access.h"
15
+
16
+ VALUE MMAPED_FILE = Qnil;
17
+
18
+ ID sym_min;
19
+ ID sym_max;
20
+ ID sym_livesum;
21
+ ID sym_gauge;
22
+ ID sym_pid;
23
+ ID sym_samples;
24
+
25
+ VALUE prom_eParsingError;
26
+
27
+ int aggregate_files(struct hashmap *map, VALUE list_of_files) {
28
+ buffer_t reading_buffer;
29
+ memset(&reading_buffer, 0, sizeof(buffer_t));
30
+
31
+ for (int i = 0; i < RARRAY_LEN(list_of_files); i++) {
32
+ VALUE params = RARRAY_PTR(list_of_files)[i];
33
+ file_t file;
34
+
35
+ if (!file_open_from_params(&file, params)) {
36
+ buffer_dispose(&reading_buffer);
37
+ return 0;
38
+ }
39
+
40
+ if (!read_from_file(&file, &reading_buffer)) {
41
+ buffer_dispose(&reading_buffer);
42
+ file_close(&file);
43
+ return 0;
44
+ }
45
+
46
+ if (!process_buffer(&file, &reading_buffer, map)) {
47
+ buffer_dispose(&reading_buffer);
48
+ file_close(&file);
49
+ return 0;
50
+ }
51
+
52
+ if (!file_close(&file)) {
53
+ buffer_dispose(&reading_buffer);
54
+ return 0;
55
+ }
56
+ }
57
+
58
+ buffer_dispose(&reading_buffer);
59
+ return 1;
60
+ }
61
+
62
+ VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list) {
63
+ struct hashmap map;
64
+ hashmap_setup(&map);
65
+
66
+ if (!aggregate_files(&map, file_list)) { // all entries in map are now copies that need to be disposed
67
+ hashmap_destroy(&map);
68
+ raise_last_exception();
69
+ return Qnil;
70
+ }
71
+
72
+ entry_t **sorted_entries;
73
+
74
+ if (!sort_map_entries(&map, &sorted_entries)) {
75
+ hashmap_destroy(&map);
76
+
77
+ raise_last_exception();
78
+ return Qnil;
79
+ }
80
+
81
+ VALUE rv = rb_str_new("", 0);
82
+ if (!entries_to_string(rv, sorted_entries, hashmap_size(&map))) {
83
+ free(sorted_entries);
84
+ hashmap_destroy(&map);
85
+
86
+ raise_last_exception();
87
+ return Qnil;
88
+ }
89
+
90
+ RB_GC_GUARD(file_list); // ensure file list is not GCed before this point
91
+ free(sorted_entries);
92
+ hashmap_destroy(&map);
93
+ return rv;
94
+ }
95
+
96
+ void Init_fast_mmaped_file() {
97
+ sym_gauge = rb_intern("gauge");
98
+ sym_min = rb_intern("min");
99
+ sym_max = rb_intern("max");
100
+ sym_livesum = rb_intern("livesum");
101
+ sym_pid = rb_intern("pid");
102
+ sym_samples = rb_intern("samples");
103
+
104
+ prom_eParsingError = rb_define_class("PrometheusParsingError", rb_eRuntimeError);
105
+
106
+ MMAPED_FILE = rb_define_class("FastMmapedFile", rb_cObject);
107
+ rb_define_const(MMAPED_FILE, "MAP_SHARED", INT2FIX(MAP_SHARED));
108
+
109
+ rb_define_singleton_method(MMAPED_FILE, "to_metrics", method_to_metrics, 1);
110
+
111
+ rb_define_alloc_func(MMAPED_FILE, mm_s_alloc);
112
+ rb_define_singleton_method(MMAPED_FILE, "new", mm_s_new, -1);
113
+ rb_define_method(MMAPED_FILE, "initialize", mm_init, 1);
114
+ rb_define_method(MMAPED_FILE, "slice", mm_aref_m, -1);
115
+ rb_define_method(MMAPED_FILE, "sync", mm_msync, -1);
116
+ rb_define_method(MMAPED_FILE, "munmap", mm_unmap, 0);
117
+
118
+ rb_define_method(MMAPED_FILE, "used", method_load_used, 0);
119
+ rb_define_method(MMAPED_FILE, "used=", method_save_used, 1);
120
+ rb_define_method(MMAPED_FILE, "fetch_entry", method_fetch_entry, 3);
121
+ rb_define_method(MMAPED_FILE, "upsert_entry", method_upsert_entry, 3);
122
+ }
@@ -0,0 +1,5 @@
1
+ #include "file_format.h"
2
+
3
+ inline uint32_t padding_length(uint32_t key_length) {
4
+ return 8 - (sizeof(uint32_t) + key_length) % 8; // padding | 8 byte aligned
5
+ }
@@ -0,0 +1,11 @@
1
+ #ifndef FILE_FORMAT_H
2
+ #define FILE_FORMAT_H
3
+
4
+ #include <stdint.h>
5
+
6
+ #define START_POSITION 8
7
+ #define INITIAL_SIZE (2 * sizeof(int32_t))
8
+
9
+ uint32_t padding_length(uint32_t key_length);
10
+
11
+ #endif
@@ -0,0 +1,195 @@
1
+ #include "file_parsing.h"
2
+
3
+ #include <hashmap.h>
4
+ #include <jsmn.h>
5
+
6
+ #include "file_format.h"
7
+ #include "globals.h"
8
+ #include "utils.h"
9
+
10
+ HASHMAP_FUNCS_CREATE(entry, const entry_t, entry_t)
11
+
12
+ typedef int (*compare_fn)(const void *a, const void *b);
13
+
14
+ static size_t hashmap_hash_entry(const entry_t *entry) { return hashmap_hash_string(entry->json); }
15
+
16
+ static int hashmap_compare_entry(const entry_t *a, const entry_t *b) {
17
+ if (a->json_size != b->json_size) {
18
+ return -1;
19
+ }
20
+
21
+ if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)) {
22
+ return -1;
23
+ }
24
+
25
+ return strncmp(a->json, b->json, a->json_size);
26
+ }
27
+
28
+ static void entry_free(entry_t *entry) {
29
+ free(entry->json);
30
+ free(entry);
31
+ }
32
+
33
+ static inline double min(double a, double b) { return a < b ? a : b; }
34
+
35
+ static inline double max(double a, double b) { return a > b ? a : b; }
36
+
37
+ static void merge_entry(entry_t *found, const entry_t *entry) {
38
+ if (entry->type == sym_gauge) {
39
+ if (entry->multiprocess_mode == sym_min) {
40
+ found->value = min(found->value, entry->value);
41
+ } else if (entry->multiprocess_mode == sym_max) {
42
+ found->value = max(found->value, entry->value);
43
+ } else if (entry->multiprocess_mode == sym_livesum) {
44
+ found->value += entry->value;
45
+ } else {
46
+ found->value = entry->value;
47
+ }
48
+ } else {
49
+ found->value += entry->value;
50
+ }
51
+ }
52
+
53
+ void merge_or_store(struct hashmap *map, entry_t *entry) {
54
+ entry_t *found = entry_hashmap_get(map, entry);
55
+ if (found) {
56
+ merge_entry(found, entry);
57
+ entry_free(entry);
58
+ } else {
59
+ entry_hashmap_put(map, entry, entry); // use the hashmap like hashset actually
60
+ }
61
+ }
62
+
63
+ entry_t *entry_new(buffer_t *source, uint32_t pos, uint32_t encoded_len, file_t *file_info) {
64
+ entry_t *entry = calloc(1, sizeof(entry_t));
65
+ if (entry == NULL) {
66
+ return NULL;
67
+ }
68
+
69
+ entry->json = malloc(encoded_len + 1);
70
+ if (entry->json == NULL) {
71
+ free(entry);
72
+ return NULL;
73
+ }
74
+
75
+ memcpy(entry->json, source->buffer + pos, encoded_len);
76
+ entry->json[encoded_len] = '\0';
77
+ entry->json_size = encoded_len;
78
+
79
+ entry->pid = file_info->pid;
80
+ entry->multiprocess_mode = file_info->multiprocess_mode;
81
+ entry->type = file_info->type;
82
+
83
+ char *value_ptr = source->buffer + pos + encoded_len + padding_length(encoded_len);
84
+ memcpy(&(entry->value), value_ptr, sizeof(double));
85
+
86
+ return entry;
87
+ }
88
+
89
+ static int add_parsed_name(entry_t *entry) {
90
+ jsmn_parser parser;
91
+ jsmn_init(&parser);
92
+
93
+ jsmntok_t tokens[2];
94
+ memset(&tokens, 0, sizeof(tokens));
95
+
96
+ jsmn_parse(&parser, entry->json, entry->json_size, tokens, 2);
97
+ jsmntok_t *name_token = &tokens[1];
98
+
99
+ if (name_token->start < name_token->end && name_token->start > 0) {
100
+ entry->name = entry->json + name_token->start;
101
+ entry->name_len = name_token->end - name_token->start;
102
+ return 1;
103
+ }
104
+ return 0;
105
+ }
106
+
107
+ static int entry_lexical_comparator(const entry_t **a, const entry_t **b) {
108
+ size_t size_a = (*a)->json_size;
109
+ size_t size_b = (*b)->json_size;
110
+ size_t min_length = size_a < size_b ? size_a : size_b;
111
+
112
+ return strncmp((*a)->json, (*b)->json, min_length);
113
+ }
114
+
115
+ void hashmap_setup(struct hashmap *map) {
116
+ hashmap_init(map, (size_t(*)(const void *))hashmap_hash_entry,
117
+ (int (*)(const void *, const void *))hashmap_compare_entry, 1000);
118
+ hashmap_set_key_alloc_funcs(map, NULL, (void (*)(void *))entry_free);
119
+ }
120
+
121
+ int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map) {
122
+ if (source->size < START_POSITION) {
123
+ // nothing to read
124
+ return 1;
125
+ }
126
+ uint32_t used;
127
+ memcpy(&used, source->buffer, sizeof(uint32_t));
128
+
129
+ if (used > source->size) {
130
+ save_exception(prom_eParsingError, "source file %s corrupted, used %u > file size %u", file_info->path, used,
131
+ source->size);
132
+ return 0;
133
+ }
134
+
135
+ uint32_t pos = START_POSITION;
136
+ while (pos + sizeof(uint32_t) < used) {
137
+ uint32_t encoded_len;
138
+ memcpy(&encoded_len, source->buffer + pos, sizeof(uint32_t));
139
+ pos += sizeof(uint32_t);
140
+
141
+ uint32_t value_offset = encoded_len + padding_length(encoded_len);
142
+
143
+ if (pos + value_offset + sizeof(double) > used) {
144
+ save_exception(prom_eParsingError, "source file %s corrupted, used %u < stored data length %u",
145
+ file_info->path, used, pos + value_offset + sizeof(double));
146
+ return 0;
147
+ }
148
+
149
+ entry_t *entry = entry_new(source, pos, encoded_len, file_info);
150
+ if (entry == NULL) {
151
+ save_exception(rb_eNoMemError, "Failed creating metrics entry");
152
+ return 0;
153
+ }
154
+
155
+ merge_or_store(map, entry);
156
+
157
+ pos += value_offset + sizeof(double);
158
+ }
159
+ return 1;
160
+ }
161
+
162
+ int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries) {
163
+ size_t num = hashmap_size(map);
164
+
165
+ entry_t **list = calloc(num, sizeof(entry_t *));
166
+
167
+ if (list == NULL) {
168
+ save_exception(rb_eNoMemError, "Couldn't allocate for %zu memory", num * sizeof(entry_t *));
169
+ return 0;
170
+ }
171
+
172
+ size_t cnt = 0;
173
+ struct hashmap_iter *iter;
174
+ for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
175
+ entry_t *entry = (entry_t *)entry_hashmap_iter_get_key(iter);
176
+ if (add_parsed_name(entry)) {
177
+ list[cnt] = entry;
178
+ cnt++;
179
+ }
180
+ }
181
+ if (cnt != num) {
182
+ save_exception(rb_eRuntimeError, "Processed entries %zu != map entries %zu", cnt, num);
183
+ free(list);
184
+ return 0;
185
+ }
186
+
187
+ qsort(list, cnt, sizeof(entry_t *), (compare_fn)&entry_lexical_comparator);
188
+ *sorted_entries = list;
189
+ return 1;
190
+ }
191
+
192
+ int is_pid_significant(const entry_t *e) {
193
+ ID mp = e->multiprocess_mode;
194
+ return e->type == sym_gauge && !(mp == sym_min || mp == sym_max || mp == sym_livesum);
195
+ }
@@ -0,0 +1,27 @@
1
+ #ifndef FILE_PARSING_H
2
+ #define FILE_PARSING_H
3
+ #include <file_reading.h>
4
+ #include <hashmap.h>
5
+ #include <ruby.h>
6
+
7
+ typedef struct {
8
+ char *json;
9
+ size_t json_size;
10
+ char *name;
11
+ size_t name_len;
12
+
13
+ ID multiprocess_mode;
14
+ ID type;
15
+ VALUE pid;
16
+
17
+ double value;
18
+ } entry_t;
19
+
20
+ void hashmap_setup(struct hashmap *map);
21
+
22
+ int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map);
23
+ int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries);
24
+
25
+ int is_pid_significant(const entry_t *e);
26
+
27
+ #endif