prometheus-client-mmap 0.20.3-x86_64-darwin
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +253 -0
- data/ext/fast_mmaped_file/extconf.rb +30 -0
- data/ext/fast_mmaped_file/fast_mmaped_file.c +122 -0
- data/ext/fast_mmaped_file/file_format.c +5 -0
- data/ext/fast_mmaped_file/file_format.h +11 -0
- data/ext/fast_mmaped_file/file_parsing.c +195 -0
- data/ext/fast_mmaped_file/file_parsing.h +27 -0
- data/ext/fast_mmaped_file/file_reading.c +102 -0
- data/ext/fast_mmaped_file/file_reading.h +30 -0
- data/ext/fast_mmaped_file/globals.h +14 -0
- data/ext/fast_mmaped_file/mmap.c +427 -0
- data/ext/fast_mmaped_file/mmap.h +61 -0
- data/ext/fast_mmaped_file/rendering.c +199 -0
- data/ext/fast_mmaped_file/rendering.h +8 -0
- data/ext/fast_mmaped_file/utils.c +56 -0
- data/ext/fast_mmaped_file/utils.h +22 -0
- data/ext/fast_mmaped_file/value_access.c +242 -0
- data/ext/fast_mmaped_file/value_access.h +15 -0
- data/ext/fast_mmaped_file_rs/.cargo/config.toml +23 -0
- data/ext/fast_mmaped_file_rs/Cargo.lock +790 -0
- data/ext/fast_mmaped_file_rs/Cargo.toml +30 -0
- data/ext/fast_mmaped_file_rs/README.md +52 -0
- data/ext/fast_mmaped_file_rs/extconf.rb +30 -0
- data/ext/fast_mmaped_file_rs/src/error.rs +174 -0
- data/ext/fast_mmaped_file_rs/src/file_entry.rs +579 -0
- data/ext/fast_mmaped_file_rs/src/file_info.rs +190 -0
- data/ext/fast_mmaped_file_rs/src/lib.rs +79 -0
- data/ext/fast_mmaped_file_rs/src/macros.rs +14 -0
- data/ext/fast_mmaped_file_rs/src/map.rs +492 -0
- data/ext/fast_mmaped_file_rs/src/mmap.rs +151 -0
- data/ext/fast_mmaped_file_rs/src/parser.rs +346 -0
- data/ext/fast_mmaped_file_rs/src/raw_entry.rs +473 -0
- data/ext/fast_mmaped_file_rs/src/testhelper.rs +222 -0
- data/ext/fast_mmaped_file_rs/src/util.rs +121 -0
- data/lib/2.7/fast_mmaped_file.bundle +0 -0
- data/lib/2.7/fast_mmaped_file_rs.bundle +0 -0
- data/lib/3.0/fast_mmaped_file.bundle +0 -0
- data/lib/3.0/fast_mmaped_file_rs.bundle +0 -0
- data/lib/3.1/fast_mmaped_file.bundle +0 -0
- data/lib/3.1/fast_mmaped_file_rs.bundle +0 -0
- data/lib/3.2/fast_mmaped_file.bundle +0 -0
- data/lib/3.2/fast_mmaped_file_rs.bundle +0 -0
- data/lib/prometheus/client/configuration.rb +23 -0
- data/lib/prometheus/client/counter.rb +27 -0
- data/lib/prometheus/client/formats/text.rb +118 -0
- data/lib/prometheus/client/gauge.rb +40 -0
- data/lib/prometheus/client/helper/entry_parser.rb +132 -0
- data/lib/prometheus/client/helper/file_locker.rb +50 -0
- data/lib/prometheus/client/helper/json_parser.rb +23 -0
- data/lib/prometheus/client/helper/metrics_processing.rb +45 -0
- data/lib/prometheus/client/helper/metrics_representation.rb +51 -0
- data/lib/prometheus/client/helper/mmaped_file.rb +64 -0
- data/lib/prometheus/client/helper/plain_file.rb +29 -0
- data/lib/prometheus/client/histogram.rb +80 -0
- data/lib/prometheus/client/label_set_validator.rb +86 -0
- data/lib/prometheus/client/metric.rb +80 -0
- data/lib/prometheus/client/mmaped_dict.rb +79 -0
- data/lib/prometheus/client/mmaped_value.rb +154 -0
- data/lib/prometheus/client/page_size.rb +17 -0
- data/lib/prometheus/client/push.rb +203 -0
- data/lib/prometheus/client/rack/collector.rb +88 -0
- data/lib/prometheus/client/rack/exporter.rb +96 -0
- data/lib/prometheus/client/registry.rb +65 -0
- data/lib/prometheus/client/simple_value.rb +31 -0
- data/lib/prometheus/client/summary.rb +69 -0
- data/lib/prometheus/client/support/unicorn.rb +35 -0
- data/lib/prometheus/client/uses_value_type.rb +20 -0
- data/lib/prometheus/client/version.rb +5 -0
- data/lib/prometheus/client.rb +58 -0
- data/lib/prometheus.rb +3 -0
- data/vendor/c/hashmap/.gitignore +52 -0
- data/vendor/c/hashmap/LICENSE +21 -0
- data/vendor/c/hashmap/README.md +90 -0
- data/vendor/c/hashmap/_config.yml +1 -0
- data/vendor/c/hashmap/src/hashmap.c +692 -0
- data/vendor/c/hashmap/src/hashmap.h +267 -0
- data/vendor/c/hashmap/test/Makefile +22 -0
- data/vendor/c/hashmap/test/hashmap_test.c +608 -0
- data/vendor/c/jsmn/.travis.yml +4 -0
- data/vendor/c/jsmn/LICENSE +20 -0
- data/vendor/c/jsmn/Makefile +41 -0
- data/vendor/c/jsmn/README.md +168 -0
- data/vendor/c/jsmn/example/jsondump.c +126 -0
- data/vendor/c/jsmn/example/simple.c +76 -0
- data/vendor/c/jsmn/jsmn.c +314 -0
- data/vendor/c/jsmn/jsmn.h +76 -0
- data/vendor/c/jsmn/library.json +16 -0
- data/vendor/c/jsmn/test/test.h +27 -0
- data/vendor/c/jsmn/test/tests.c +407 -0
- data/vendor/c/jsmn/test/testutil.h +94 -0
- metadata +243 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 0cb22da1457bb5f77b2a2cab6bf3fdf3d5925cbad8d2f2ca2d336c0a34077cb1
|
4
|
+
data.tar.gz: b9a2b9185907476e6ce606cccd9ba2e7c9a3af162db82310b08deb0cfaf10387
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: be6777a5c65f8157aa2d0b10d02b07f1334ee416293337e59382644e5603cac40800fa889cd3aa69930556ffb6e06be897b5759b194795ca129dac91009faf3b
|
7
|
+
data.tar.gz: 0fa2b4f65b105492141c6078fa0e0bdfd2050a73d16c6e322f238b85b2c52a9ecf9e03b529d78381d3b4621df2b98824a96c632f15258a0809de4c18fffb0393
|
data/README.md
ADDED
@@ -0,0 +1,253 @@
|
|
1
|
+
# Prometheus Ruby Mmap Client
|
2
|
+
|
3
|
+
This Prometheus library is fork of [Prometheus Ruby Client](https://github.com/prometheus/client_ruby)
|
4
|
+
that uses mmap'ed files to share metrics from multiple processes.
|
5
|
+
This allows efficient metrics processing for Ruby web apps running in multiprocess setups like Unicorn.
|
6
|
+
|
7
|
+
A suite of instrumentation metric primitives for Ruby that can be exposed
|
8
|
+
through a HTTP interface. Intended to be used together with a
|
9
|
+
[Prometheus server][1].
|
10
|
+
|
11
|
+
[![Gem Version][4]](http://badge.fury.io/rb/prometheus-client-mmap)
|
12
|
+
[![Build Status][3]](https://gitlab.com/gitlab-org/prometheus-client-mmap/commits/master)
|
13
|
+
[![Dependency Status][5]](https://gemnasium.com/prometheus/prometheus-client-mmap)
|
14
|
+
|
15
|
+
## Usage
|
16
|
+
|
17
|
+
### Overview
|
18
|
+
|
19
|
+
```ruby
|
20
|
+
require 'prometheus/client'
|
21
|
+
|
22
|
+
# returns a default registry
|
23
|
+
prometheus = Prometheus::Client.registry
|
24
|
+
|
25
|
+
# create a new counter metric
|
26
|
+
http_requests = Prometheus::Client::Counter.new(:http_requests, 'A counter of HTTP requests made')
|
27
|
+
# register the metric
|
28
|
+
prometheus.register(http_requests)
|
29
|
+
|
30
|
+
# equivalent helper function
|
31
|
+
http_requests = prometheus.counter(:http_requests, 'A counter of HTTP requests made')
|
32
|
+
|
33
|
+
# start using the counter
|
34
|
+
http_requests.increment
|
35
|
+
```
|
36
|
+
|
37
|
+
## Rust extension (experimental)
|
38
|
+
|
39
|
+
In an effort to improve maintainability, there is now an optional Rust
|
40
|
+
implementation that reads the metric files and outputs the multiprocess
|
41
|
+
metrics to text. If `rustc` is available, then the Rust extension will
|
42
|
+
be built automatically. The `use_rust` keyword argument can be used:
|
43
|
+
|
44
|
+
```ruby
|
45
|
+
puts Prometheus::Client::Formats::Text.marshal_multiprocess(use_rust: true)
|
46
|
+
```
|
47
|
+
|
48
|
+
Note that this parameter will likely be deprecated and removed once the Rust
|
49
|
+
extension becomes the default mode.
|
50
|
+
|
51
|
+
### Rack middleware
|
52
|
+
|
53
|
+
There are two [Rack][2] middlewares available, one to expose a metrics HTTP
|
54
|
+
endpoint to be scraped by a prometheus server ([Exporter][9]) and one to trace all HTTP
|
55
|
+
requests ([Collector][10]).
|
56
|
+
|
57
|
+
It's highly recommended to enable gzip compression for the metrics endpoint,
|
58
|
+
for example by including the `Rack::Deflater` middleware.
|
59
|
+
|
60
|
+
```ruby
|
61
|
+
# config.ru
|
62
|
+
|
63
|
+
require 'rack'
|
64
|
+
require 'prometheus/client/rack/collector'
|
65
|
+
require 'prometheus/client/rack/exporter'
|
66
|
+
|
67
|
+
use Rack::Deflater, if: ->(env, status, headers, body) { body.any? && body[0].length > 512 }
|
68
|
+
use Prometheus::Client::Rack::Collector
|
69
|
+
use Prometheus::Client::Rack::Exporter
|
70
|
+
|
71
|
+
run ->(env) { [200, {'Content-Type' => 'text/html'}, ['OK']] }
|
72
|
+
```
|
73
|
+
|
74
|
+
Start the server and have a look at the metrics endpoint:
|
75
|
+
[http://localhost:5000/metrics](http://localhost:5000/metrics).
|
76
|
+
|
77
|
+
For further instructions and other scripts to get started, have a look at the
|
78
|
+
integrated [example application](examples/rack/README.md).
|
79
|
+
|
80
|
+
### Pushgateway
|
81
|
+
|
82
|
+
The Ruby client can also be used to push its collected metrics to a
|
83
|
+
[Pushgateway][8]. This comes in handy with batch jobs or in other scenarios
|
84
|
+
where it's not possible or feasible to let a Prometheus server scrape a Ruby
|
85
|
+
process. TLS and HTTP basic authentication are supported.
|
86
|
+
|
87
|
+
```ruby
|
88
|
+
require 'prometheus/client'
|
89
|
+
require 'prometheus/client/push'
|
90
|
+
|
91
|
+
registry = Prometheus::Client.registry
|
92
|
+
# ... register some metrics, set/increment/observe/etc. their values
|
93
|
+
|
94
|
+
# push the registry state to the default gateway
|
95
|
+
Prometheus::Client::Push.new(job: 'my-batch-job').add(registry)
|
96
|
+
|
97
|
+
# optional: specify a grouping key that uniquely identifies a job instance, and gateway.
|
98
|
+
#
|
99
|
+
# Note: the labels you use in the grouping key must not conflict with labels set on the
|
100
|
+
# metrics being pushed. If they do, an error will be raised.
|
101
|
+
Prometheus::Client::Push.new(
|
102
|
+
job: 'my-batch-job',
|
103
|
+
gateway: 'https://example.domain:1234',
|
104
|
+
grouping_key: { instance: 'some-instance', extra_key: 'foobar' }
|
105
|
+
).add(registry)
|
106
|
+
|
107
|
+
# If you want to replace any previously pushed metrics for a given grouping key,
|
108
|
+
# use the #replace method.
|
109
|
+
#
|
110
|
+
# Unlike #add, this will completely replace the metrics under the specified grouping key
|
111
|
+
# (i.e. anything currently present in the pushgateway for the specified grouping key, but
|
112
|
+
# not present in the registry for that grouping key will be removed).
|
113
|
+
#
|
114
|
+
# See https://github.com/prometheus/pushgateway#put-method for a full explanation.
|
115
|
+
Prometheus::Client::Push.new(job: 'my-batch-job').replace(registry)
|
116
|
+
|
117
|
+
# If you want to delete all previously pushed metrics for a given grouping key,
|
118
|
+
# use the #delete method.
|
119
|
+
Prometheus::Client::Push.new(job: 'my-batch-job').delete
|
120
|
+
```
|
121
|
+
|
122
|
+
## Metrics
|
123
|
+
|
124
|
+
The following metric types are currently supported.
|
125
|
+
|
126
|
+
### Counter
|
127
|
+
|
128
|
+
Counter is a metric that exposes merely a sum or tally of things.
|
129
|
+
|
130
|
+
```ruby
|
131
|
+
counter = Prometheus::Client::Counter.new(:service_requests_total, '...')
|
132
|
+
|
133
|
+
# increment the counter for a given label set
|
134
|
+
counter.increment({ service: 'foo' })
|
135
|
+
|
136
|
+
# increment by a given value
|
137
|
+
counter.increment({ service: 'bar' }, 5)
|
138
|
+
|
139
|
+
# get current value for a given label set
|
140
|
+
counter.get({ service: 'bar' })
|
141
|
+
# => 5
|
142
|
+
```
|
143
|
+
|
144
|
+
### Gauge
|
145
|
+
|
146
|
+
Gauge is a metric that exposes merely an instantaneous value or some snapshot
|
147
|
+
thereof.
|
148
|
+
|
149
|
+
```ruby
|
150
|
+
gauge = Prometheus::Client::Gauge.new(:room_temperature_celsius, '...')
|
151
|
+
|
152
|
+
# set a value
|
153
|
+
gauge.set({ room: 'kitchen' }, 21.534)
|
154
|
+
|
155
|
+
# retrieve the current value for a given label set
|
156
|
+
gauge.get({ room: 'kitchen' })
|
157
|
+
# => 21.534
|
158
|
+
```
|
159
|
+
|
160
|
+
### Histogram
|
161
|
+
|
162
|
+
A histogram samples observations (usually things like request durations or
|
163
|
+
response sizes) and counts them in configurable buckets. It also provides a sum
|
164
|
+
of all observed values.
|
165
|
+
|
166
|
+
```ruby
|
167
|
+
histogram = Prometheus::Client::Histogram.new(:service_latency_seconds, '...')
|
168
|
+
|
169
|
+
# record a value
|
170
|
+
histogram.observe({ service: 'users' }, Benchmark.realtime { service.call(arg) })
|
171
|
+
|
172
|
+
# retrieve the current bucket values
|
173
|
+
histogram.get({ service: 'users' })
|
174
|
+
# => { 0.005 => 3, 0.01 => 15, 0.025 => 18, ..., 2.5 => 42, 5 => 42, 10 = >42 }
|
175
|
+
```
|
176
|
+
|
177
|
+
### Summary
|
178
|
+
|
179
|
+
Summary, similar to histograms, is an accumulator for samples. It captures
|
180
|
+
Numeric data and provides an efficient percentile calculation mechanism.
|
181
|
+
|
182
|
+
```ruby
|
183
|
+
summary = Prometheus::Client::Summary.new(:service_latency_seconds, '...')
|
184
|
+
|
185
|
+
# record a value
|
186
|
+
summary.observe({ service: 'database' }, Benchmark.realtime { service.call() })
|
187
|
+
|
188
|
+
# retrieve the current quantile values
|
189
|
+
summary.get({ service: 'database' })
|
190
|
+
# => { 0.5 => 0.1233122, 0.9 => 3.4323, 0.99 => 5.3428231 }
|
191
|
+
```
|
192
|
+
|
193
|
+
## Configuration
|
194
|
+
|
195
|
+
### Memory mapped files storage location
|
196
|
+
|
197
|
+
Set `prometheus_multiproc_dir` environment variable to the path where you want metric files to be stored. Example:
|
198
|
+
|
199
|
+
```
|
200
|
+
prometheus_multiproc_dir=/tmp
|
201
|
+
```
|
202
|
+
|
203
|
+
## Pitfalls
|
204
|
+
|
205
|
+
### PID cardinality
|
206
|
+
|
207
|
+
In multiprocess setup e.g. running under Unicorn, having worker process restart often can
|
208
|
+
lead to performance problems when proccesing metric files. By default each process using
|
209
|
+
Prometheus metrics will create a set of files based on that process PID. With high worker
|
210
|
+
churn this will lead to creation of thousands of files and in turn will cause very noticable
|
211
|
+
slowdown when displaying metrics
|
212
|
+
|
213
|
+
To reduce this problem, a surrogate process id can be used. Set of all such IDs needs
|
214
|
+
have low cardinality, and each process id must be unique among all running process.
|
215
|
+
|
216
|
+
For Unicorn a worker id/number can be used to greatly speedup the metrics rendering.
|
217
|
+
|
218
|
+
To use it add this line to your `configure` block:
|
219
|
+
|
220
|
+
```ruby
|
221
|
+
config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider)
|
222
|
+
```
|
223
|
+
|
224
|
+
## Tools
|
225
|
+
|
226
|
+
###`bin/parse`
|
227
|
+
|
228
|
+
This command can be used to parse metric files located on the filesystem just like a metric exporter would.
|
229
|
+
It outputs either `json` formatted raw data or digested data in prometheus `text` format.
|
230
|
+
|
231
|
+
#### Usage:
|
232
|
+
|
233
|
+
```bash
|
234
|
+
$ ./bin/parse -h
|
235
|
+
Usage: parse [options] files...
|
236
|
+
-t, --to-prometheus-text format output using Prometheus text formatter
|
237
|
+
-p, --profile enable profiling
|
238
|
+
-h, --help Show this message
|
239
|
+
```
|
240
|
+
|
241
|
+
## Development
|
242
|
+
|
243
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
244
|
+
|
245
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
246
|
+
|
247
|
+
[1]: https://github.com/prometheus/prometheus
|
248
|
+
[2]: http://rack.github.io/
|
249
|
+
[3]: https://gitlab.com/gitlab-org/prometheus-client-mmap/badges/master/pipeline.svg
|
250
|
+
[4]: https://badge.fury.io/rb/prometheus-client.svg
|
251
|
+
[8]: https://github.com/prometheus/pushgateway
|
252
|
+
[9]: lib/prometheus/client/rack/exporter.rb
|
253
|
+
[10]: lib/prometheus/client/rack/collector.rb
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'mkmf'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
$CFLAGS << ' -std=c99 -D_POSIX_C_SOURCE=200809L -Wall -Wextra'
|
5
|
+
|
6
|
+
if enable_config('fail-on-warning')
|
7
|
+
$CFLAGS << ' -Werror'
|
8
|
+
end
|
9
|
+
|
10
|
+
if enable_config('debug')
|
11
|
+
$CFLAGS << ' -O0 -g'
|
12
|
+
end
|
13
|
+
|
14
|
+
if enable_config('address-sanitizer')
|
15
|
+
$CFLAGS << ' -O -fsanitize=address -fno-omit-frame-pointer -g'
|
16
|
+
end
|
17
|
+
|
18
|
+
CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
|
19
|
+
|
20
|
+
cwd = File.expand_path(File.dirname(__FILE__))
|
21
|
+
vendor_dir = File.join(cwd, '../../vendor/c')
|
22
|
+
src_dir = File.join(cwd, '../../ext/fast_mmaped_file')
|
23
|
+
|
24
|
+
src_files = %W[#{vendor_dir}/jsmn/jsmn.c #{vendor_dir}/hashmap/src/hashmap.c]
|
25
|
+
FileUtils.cp(src_files, src_dir)
|
26
|
+
|
27
|
+
$INCFLAGS << " -I#{vendor_dir}/jsmn -I#{vendor_dir}/hashmap/src"
|
28
|
+
|
29
|
+
dir_config('fast_mmaped_file')
|
30
|
+
create_makefile('fast_mmaped_file')
|
@@ -0,0 +1,122 @@
|
|
1
|
+
#include <errno.h>
|
2
|
+
#include <hashmap.h>
|
3
|
+
#include <jsmn.h>
|
4
|
+
#include <ruby.h>
|
5
|
+
#include <ruby/intern.h>
|
6
|
+
#include <sys/mman.h>
|
7
|
+
|
8
|
+
#include "file_parsing.h"
|
9
|
+
#include "file_reading.h"
|
10
|
+
#include "globals.h"
|
11
|
+
#include "mmap.h"
|
12
|
+
#include "rendering.h"
|
13
|
+
#include "utils.h"
|
14
|
+
#include "value_access.h"
|
15
|
+
|
16
|
+
VALUE MMAPED_FILE = Qnil;
|
17
|
+
|
18
|
+
ID sym_min;
|
19
|
+
ID sym_max;
|
20
|
+
ID sym_livesum;
|
21
|
+
ID sym_gauge;
|
22
|
+
ID sym_pid;
|
23
|
+
ID sym_samples;
|
24
|
+
|
25
|
+
VALUE prom_eParsingError;
|
26
|
+
|
27
|
+
int aggregate_files(struct hashmap *map, VALUE list_of_files) {
|
28
|
+
buffer_t reading_buffer;
|
29
|
+
memset(&reading_buffer, 0, sizeof(buffer_t));
|
30
|
+
|
31
|
+
for (int i = 0; i < RARRAY_LEN(list_of_files); i++) {
|
32
|
+
VALUE params = RARRAY_PTR(list_of_files)[i];
|
33
|
+
file_t file;
|
34
|
+
|
35
|
+
if (!file_open_from_params(&file, params)) {
|
36
|
+
buffer_dispose(&reading_buffer);
|
37
|
+
return 0;
|
38
|
+
}
|
39
|
+
|
40
|
+
if (!read_from_file(&file, &reading_buffer)) {
|
41
|
+
buffer_dispose(&reading_buffer);
|
42
|
+
file_close(&file);
|
43
|
+
return 0;
|
44
|
+
}
|
45
|
+
|
46
|
+
if (!process_buffer(&file, &reading_buffer, map)) {
|
47
|
+
buffer_dispose(&reading_buffer);
|
48
|
+
file_close(&file);
|
49
|
+
return 0;
|
50
|
+
}
|
51
|
+
|
52
|
+
if (!file_close(&file)) {
|
53
|
+
buffer_dispose(&reading_buffer);
|
54
|
+
return 0;
|
55
|
+
}
|
56
|
+
}
|
57
|
+
|
58
|
+
buffer_dispose(&reading_buffer);
|
59
|
+
return 1;
|
60
|
+
}
|
61
|
+
|
62
|
+
VALUE method_to_metrics(VALUE UNUSED(self), VALUE file_list) {
|
63
|
+
struct hashmap map;
|
64
|
+
hashmap_setup(&map);
|
65
|
+
|
66
|
+
if (!aggregate_files(&map, file_list)) { // all entries in map are now copies that need to be disposed
|
67
|
+
hashmap_destroy(&map);
|
68
|
+
raise_last_exception();
|
69
|
+
return Qnil;
|
70
|
+
}
|
71
|
+
|
72
|
+
entry_t **sorted_entries;
|
73
|
+
|
74
|
+
if (!sort_map_entries(&map, &sorted_entries)) {
|
75
|
+
hashmap_destroy(&map);
|
76
|
+
|
77
|
+
raise_last_exception();
|
78
|
+
return Qnil;
|
79
|
+
}
|
80
|
+
|
81
|
+
VALUE rv = rb_str_new("", 0);
|
82
|
+
if (!entries_to_string(rv, sorted_entries, hashmap_size(&map))) {
|
83
|
+
free(sorted_entries);
|
84
|
+
hashmap_destroy(&map);
|
85
|
+
|
86
|
+
raise_last_exception();
|
87
|
+
return Qnil;
|
88
|
+
}
|
89
|
+
|
90
|
+
RB_GC_GUARD(file_list); // ensure file list is not GCed before this point
|
91
|
+
free(sorted_entries);
|
92
|
+
hashmap_destroy(&map);
|
93
|
+
return rv;
|
94
|
+
}
|
95
|
+
|
96
|
+
void Init_fast_mmaped_file() {
|
97
|
+
sym_gauge = rb_intern("gauge");
|
98
|
+
sym_min = rb_intern("min");
|
99
|
+
sym_max = rb_intern("max");
|
100
|
+
sym_livesum = rb_intern("livesum");
|
101
|
+
sym_pid = rb_intern("pid");
|
102
|
+
sym_samples = rb_intern("samples");
|
103
|
+
|
104
|
+
prom_eParsingError = rb_define_class("PrometheusParsingError", rb_eRuntimeError);
|
105
|
+
|
106
|
+
MMAPED_FILE = rb_define_class("FastMmapedFile", rb_cObject);
|
107
|
+
rb_define_const(MMAPED_FILE, "MAP_SHARED", INT2FIX(MAP_SHARED));
|
108
|
+
|
109
|
+
rb_define_singleton_method(MMAPED_FILE, "to_metrics", method_to_metrics, 1);
|
110
|
+
|
111
|
+
rb_define_alloc_func(MMAPED_FILE, mm_s_alloc);
|
112
|
+
rb_define_singleton_method(MMAPED_FILE, "new", mm_s_new, -1);
|
113
|
+
rb_define_method(MMAPED_FILE, "initialize", mm_init, 1);
|
114
|
+
rb_define_method(MMAPED_FILE, "slice", mm_aref_m, -1);
|
115
|
+
rb_define_method(MMAPED_FILE, "sync", mm_msync, -1);
|
116
|
+
rb_define_method(MMAPED_FILE, "munmap", mm_unmap, 0);
|
117
|
+
|
118
|
+
rb_define_method(MMAPED_FILE, "used", method_load_used, 0);
|
119
|
+
rb_define_method(MMAPED_FILE, "used=", method_save_used, 1);
|
120
|
+
rb_define_method(MMAPED_FILE, "fetch_entry", method_fetch_entry, 3);
|
121
|
+
rb_define_method(MMAPED_FILE, "upsert_entry", method_upsert_entry, 3);
|
122
|
+
}
|
@@ -0,0 +1,195 @@
|
|
1
|
+
#include "file_parsing.h"
|
2
|
+
|
3
|
+
#include <hashmap.h>
|
4
|
+
#include <jsmn.h>
|
5
|
+
|
6
|
+
#include "file_format.h"
|
7
|
+
#include "globals.h"
|
8
|
+
#include "utils.h"
|
9
|
+
|
10
|
+
HASHMAP_FUNCS_CREATE(entry, const entry_t, entry_t)
|
11
|
+
|
12
|
+
typedef int (*compare_fn)(const void *a, const void *b);
|
13
|
+
|
14
|
+
static size_t hashmap_hash_entry(const entry_t *entry) { return hashmap_hash_string(entry->json); }
|
15
|
+
|
16
|
+
static int hashmap_compare_entry(const entry_t *a, const entry_t *b) {
|
17
|
+
if (a->json_size != b->json_size) {
|
18
|
+
return -1;
|
19
|
+
}
|
20
|
+
|
21
|
+
if (is_pid_significant(a) && (rb_str_equal(a->pid, b->pid) == Qfalse)) {
|
22
|
+
return -1;
|
23
|
+
}
|
24
|
+
|
25
|
+
return strncmp(a->json, b->json, a->json_size);
|
26
|
+
}
|
27
|
+
|
28
|
+
static void entry_free(entry_t *entry) {
|
29
|
+
free(entry->json);
|
30
|
+
free(entry);
|
31
|
+
}
|
32
|
+
|
33
|
+
static inline double min(double a, double b) { return a < b ? a : b; }
|
34
|
+
|
35
|
+
static inline double max(double a, double b) { return a > b ? a : b; }
|
36
|
+
|
37
|
+
static void merge_entry(entry_t *found, const entry_t *entry) {
|
38
|
+
if (entry->type == sym_gauge) {
|
39
|
+
if (entry->multiprocess_mode == sym_min) {
|
40
|
+
found->value = min(found->value, entry->value);
|
41
|
+
} else if (entry->multiprocess_mode == sym_max) {
|
42
|
+
found->value = max(found->value, entry->value);
|
43
|
+
} else if (entry->multiprocess_mode == sym_livesum) {
|
44
|
+
found->value += entry->value;
|
45
|
+
} else {
|
46
|
+
found->value = entry->value;
|
47
|
+
}
|
48
|
+
} else {
|
49
|
+
found->value += entry->value;
|
50
|
+
}
|
51
|
+
}
|
52
|
+
|
53
|
+
void merge_or_store(struct hashmap *map, entry_t *entry) {
|
54
|
+
entry_t *found = entry_hashmap_get(map, entry);
|
55
|
+
if (found) {
|
56
|
+
merge_entry(found, entry);
|
57
|
+
entry_free(entry);
|
58
|
+
} else {
|
59
|
+
entry_hashmap_put(map, entry, entry); // use the hashmap like hashset actually
|
60
|
+
}
|
61
|
+
}
|
62
|
+
|
63
|
+
entry_t *entry_new(buffer_t *source, uint32_t pos, uint32_t encoded_len, file_t *file_info) {
|
64
|
+
entry_t *entry = calloc(1, sizeof(entry_t));
|
65
|
+
if (entry == NULL) {
|
66
|
+
return NULL;
|
67
|
+
}
|
68
|
+
|
69
|
+
entry->json = malloc(encoded_len + 1);
|
70
|
+
if (entry->json == NULL) {
|
71
|
+
free(entry);
|
72
|
+
return NULL;
|
73
|
+
}
|
74
|
+
|
75
|
+
memcpy(entry->json, source->buffer + pos, encoded_len);
|
76
|
+
entry->json[encoded_len] = '\0';
|
77
|
+
entry->json_size = encoded_len;
|
78
|
+
|
79
|
+
entry->pid = file_info->pid;
|
80
|
+
entry->multiprocess_mode = file_info->multiprocess_mode;
|
81
|
+
entry->type = file_info->type;
|
82
|
+
|
83
|
+
char *value_ptr = source->buffer + pos + encoded_len + padding_length(encoded_len);
|
84
|
+
memcpy(&(entry->value), value_ptr, sizeof(double));
|
85
|
+
|
86
|
+
return entry;
|
87
|
+
}
|
88
|
+
|
89
|
+
static int add_parsed_name(entry_t *entry) {
|
90
|
+
jsmn_parser parser;
|
91
|
+
jsmn_init(&parser);
|
92
|
+
|
93
|
+
jsmntok_t tokens[2];
|
94
|
+
memset(&tokens, 0, sizeof(tokens));
|
95
|
+
|
96
|
+
jsmn_parse(&parser, entry->json, entry->json_size, tokens, 2);
|
97
|
+
jsmntok_t *name_token = &tokens[1];
|
98
|
+
|
99
|
+
if (name_token->start < name_token->end && name_token->start > 0) {
|
100
|
+
entry->name = entry->json + name_token->start;
|
101
|
+
entry->name_len = name_token->end - name_token->start;
|
102
|
+
return 1;
|
103
|
+
}
|
104
|
+
return 0;
|
105
|
+
}
|
106
|
+
|
107
|
+
static int entry_lexical_comparator(const entry_t **a, const entry_t **b) {
|
108
|
+
size_t size_a = (*a)->json_size;
|
109
|
+
size_t size_b = (*b)->json_size;
|
110
|
+
size_t min_length = size_a < size_b ? size_a : size_b;
|
111
|
+
|
112
|
+
return strncmp((*a)->json, (*b)->json, min_length);
|
113
|
+
}
|
114
|
+
|
115
|
+
void hashmap_setup(struct hashmap *map) {
|
116
|
+
hashmap_init(map, (size_t(*)(const void *))hashmap_hash_entry,
|
117
|
+
(int (*)(const void *, const void *))hashmap_compare_entry, 1000);
|
118
|
+
hashmap_set_key_alloc_funcs(map, NULL, (void (*)(void *))entry_free);
|
119
|
+
}
|
120
|
+
|
121
|
+
int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map) {
|
122
|
+
if (source->size < START_POSITION) {
|
123
|
+
// nothing to read
|
124
|
+
return 1;
|
125
|
+
}
|
126
|
+
uint32_t used;
|
127
|
+
memcpy(&used, source->buffer, sizeof(uint32_t));
|
128
|
+
|
129
|
+
if (used > source->size) {
|
130
|
+
save_exception(prom_eParsingError, "source file %s corrupted, used %u > file size %u", file_info->path, used,
|
131
|
+
source->size);
|
132
|
+
return 0;
|
133
|
+
}
|
134
|
+
|
135
|
+
uint32_t pos = START_POSITION;
|
136
|
+
while (pos + sizeof(uint32_t) < used) {
|
137
|
+
uint32_t encoded_len;
|
138
|
+
memcpy(&encoded_len, source->buffer + pos, sizeof(uint32_t));
|
139
|
+
pos += sizeof(uint32_t);
|
140
|
+
|
141
|
+
uint32_t value_offset = encoded_len + padding_length(encoded_len);
|
142
|
+
|
143
|
+
if (pos + value_offset + sizeof(double) > used) {
|
144
|
+
save_exception(prom_eParsingError, "source file %s corrupted, used %u < stored data length %u",
|
145
|
+
file_info->path, used, pos + value_offset + sizeof(double));
|
146
|
+
return 0;
|
147
|
+
}
|
148
|
+
|
149
|
+
entry_t *entry = entry_new(source, pos, encoded_len, file_info);
|
150
|
+
if (entry == NULL) {
|
151
|
+
save_exception(rb_eNoMemError, "Failed creating metrics entry");
|
152
|
+
return 0;
|
153
|
+
}
|
154
|
+
|
155
|
+
merge_or_store(map, entry);
|
156
|
+
|
157
|
+
pos += value_offset + sizeof(double);
|
158
|
+
}
|
159
|
+
return 1;
|
160
|
+
}
|
161
|
+
|
162
|
+
int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries) {
|
163
|
+
size_t num = hashmap_size(map);
|
164
|
+
|
165
|
+
entry_t **list = calloc(num, sizeof(entry_t *));
|
166
|
+
|
167
|
+
if (list == NULL) {
|
168
|
+
save_exception(rb_eNoMemError, "Couldn't allocate for %zu memory", num * sizeof(entry_t *));
|
169
|
+
return 0;
|
170
|
+
}
|
171
|
+
|
172
|
+
size_t cnt = 0;
|
173
|
+
struct hashmap_iter *iter;
|
174
|
+
for (iter = hashmap_iter(map); iter; iter = hashmap_iter_next(map, iter)) {
|
175
|
+
entry_t *entry = (entry_t *)entry_hashmap_iter_get_key(iter);
|
176
|
+
if (add_parsed_name(entry)) {
|
177
|
+
list[cnt] = entry;
|
178
|
+
cnt++;
|
179
|
+
}
|
180
|
+
}
|
181
|
+
if (cnt != num) {
|
182
|
+
save_exception(rb_eRuntimeError, "Processed entries %zu != map entries %zu", cnt, num);
|
183
|
+
free(list);
|
184
|
+
return 0;
|
185
|
+
}
|
186
|
+
|
187
|
+
qsort(list, cnt, sizeof(entry_t *), (compare_fn)&entry_lexical_comparator);
|
188
|
+
*sorted_entries = list;
|
189
|
+
return 1;
|
190
|
+
}
|
191
|
+
|
192
|
+
int is_pid_significant(const entry_t *e) {
|
193
|
+
ID mp = e->multiprocess_mode;
|
194
|
+
return e->type == sym_gauge && !(mp == sym_min || mp == sym_max || mp == sym_livesum);
|
195
|
+
}
|
@@ -0,0 +1,27 @@
|
|
1
|
+
#ifndef FILE_PARSING_H
|
2
|
+
#define FILE_PARSING_H
|
3
|
+
#include <file_reading.h>
|
4
|
+
#include <hashmap.h>
|
5
|
+
#include <ruby.h>
|
6
|
+
|
7
|
+
typedef struct {
|
8
|
+
char *json;
|
9
|
+
size_t json_size;
|
10
|
+
char *name;
|
11
|
+
size_t name_len;
|
12
|
+
|
13
|
+
ID multiprocess_mode;
|
14
|
+
ID type;
|
15
|
+
VALUE pid;
|
16
|
+
|
17
|
+
double value;
|
18
|
+
} entry_t;
|
19
|
+
|
20
|
+
void hashmap_setup(struct hashmap *map);
|
21
|
+
|
22
|
+
int process_buffer(file_t *file_info, buffer_t *source, struct hashmap *map);
|
23
|
+
int sort_map_entries(const struct hashmap *map, entry_t ***sorted_entries);
|
24
|
+
|
25
|
+
int is_pid_significant(const entry_t *e);
|
26
|
+
|
27
|
+
#endif
|