prometheus-client-mmap 0.7.0.beta12 → 0.7.0.beta13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +21 -0
- data/lib/prometheus/client.rb +4 -0
- data/lib/prometheus/client/configuration.rb +2 -1
- data/lib/prometheus/client/formats/text.rb +18 -22
- data/lib/prometheus/client/helper/json_parser.rb +19 -0
- data/lib/prometheus/client/mmaped_dict.rb +35 -22
- data/lib/prometheus/client/mmaped_value.rb +7 -6
- data/lib/prometheus/client/support/unicorn.rb +24 -0
- data/lib/prometheus/client/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6fcba3cc4377e6b167f11073280a999e4df75717
|
4
|
+
data.tar.gz: 9635faeeb5c2fb643176954f417acc738ee60756
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 85ebce9a92214fb77be47134d3282a5a71661e9c0c8d2140f6aeed8abc00bf6780541ca33cb67f2394047137740f27098725406668eda36d4b1091ae4fbec126
|
7
|
+
data.tar.gz: 259e6ea42822d509900da3c97602774e4b69cf1384439202d53924fc12d16d7c814298adeeeefd7a553027f66b028601fe9be16701da3680f4fb14c7cb364932
|
data/README.md
CHANGED
@@ -164,6 +164,27 @@ summary.get({ service: 'database' })
|
|
164
164
|
# => { 0.5 => 0.1233122, 0.9 => 3.4323, 0.99 => 5.3428231 }
|
165
165
|
```
|
166
166
|
|
167
|
+
## Pitfalls
|
168
|
+
|
169
|
+
### PID cardinality
|
170
|
+
|
171
|
+
In multiprocess setup e.g. running under Unicorn, having worker process restart often can
|
172
|
+
lead to performance problems when proccesing metric files. By default each process using
|
173
|
+
Prometheus metrics will create a set of files based on that process PID. With high worker
|
174
|
+
churn this will lead to creation of thousands of files and in turn will cause very noticable
|
175
|
+
slowdown when displaying metrics
|
176
|
+
|
177
|
+
To reduce this problem, a surrogate process id can be used. Set of all such IDs needs
|
178
|
+
have low cardinality, and each process id must be unique among all running process.
|
179
|
+
|
180
|
+
For Unicorn a worker id/number can be used to greatly speedup the metrics rendering.
|
181
|
+
|
182
|
+
To use it add this line to your `configure` block:
|
183
|
+
|
184
|
+
```ruby
|
185
|
+
config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider)
|
186
|
+
```
|
187
|
+
|
167
188
|
## Tests
|
168
189
|
|
169
190
|
Install necessary development gems with `bundle install` and run tests with
|
data/lib/prometheus/client.rb
CHANGED
@@ -5,13 +5,14 @@ require 'logger'
|
|
5
5
|
module Prometheus
|
6
6
|
module Client
|
7
7
|
class Configuration
|
8
|
-
attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger
|
8
|
+
attr_accessor :value_class, :multiprocess_files_dir, :initial_mmap_file_size, :logger, :pid_provider
|
9
9
|
|
10
10
|
def initialize
|
11
11
|
@value_class = ::Prometheus::Client::MmapedValue
|
12
12
|
@multiprocess_files_dir = ENV['prometheus_multiproc_dir']
|
13
13
|
@initial_mmap_file_size = 4 * 1024
|
14
14
|
@logger = Logger.new($stdout)
|
15
|
+
@pid_provider = Process.method(:pid)
|
15
16
|
end
|
16
17
|
end
|
17
18
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'prometheus/client/uses_value_type'
|
2
|
+
require 'prometheus/client/helper/json_parser'
|
2
3
|
|
3
4
|
module Prometheus
|
4
5
|
module Client
|
@@ -96,29 +97,24 @@ module Prometheus
|
|
96
97
|
Dir.glob(File.join(path, '*.db')).sort.each do |f|
|
97
98
|
parts = File.basename(f, '.db').split('_')
|
98
99
|
type = parts[0].to_sym
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
# The duplicates and labels are fixed in the next for.
|
116
|
-
metric[:samples] += [[name, labelnames.zip(labelvalues), value]]
|
117
|
-
end
|
118
|
-
metrics[metric_name] = metric
|
100
|
+
|
101
|
+
MmapedDict.read_all_values(f).each do |key, value|
|
102
|
+
metric_name, name, labelnames, labelvalues = Helper::JsonParser.load(key)
|
103
|
+
metric = metrics.fetch(metric_name,
|
104
|
+
metric_name: metric_name,
|
105
|
+
help: 'Multiprocess metric',
|
106
|
+
type: type,
|
107
|
+
samples: []
|
108
|
+
)
|
109
|
+
if type == :gauge
|
110
|
+
pid = parts[2]
|
111
|
+
metric[:multiprocess_mode] = parts[1]
|
112
|
+
metric[:samples] += [[name, labelnames.zip(labelvalues) + [['pid', pid]], value]]
|
113
|
+
else
|
114
|
+
# The duplicates and labels are fixed in the next for.
|
115
|
+
metric[:samples] += [[name, labelnames.zip(labelvalues), value]]
|
119
116
|
end
|
120
|
-
|
121
|
-
d.close
|
117
|
+
metrics[metric_name] = metric
|
122
118
|
end
|
123
119
|
end
|
124
120
|
metrics
|
@@ -3,7 +3,8 @@ require 'mmap'
|
|
3
3
|
|
4
4
|
module Prometheus
|
5
5
|
module Client
|
6
|
-
class ParsingError < StandardError
|
6
|
+
class ParsingError < StandardError
|
7
|
+
end
|
7
8
|
|
8
9
|
# A dict of doubles, backed by an mmapped file.
|
9
10
|
#
|
@@ -23,30 +24,45 @@ module Prometheus
|
|
23
24
|
@f = File.open(filename, 'a+b')
|
24
25
|
process_file
|
25
26
|
rescue StandardError => e
|
26
|
-
raise ParsingError
|
27
|
+
raise ParsingError, "exception #{e} while processing metrics file #{@f.path}"
|
27
28
|
end
|
28
29
|
|
29
|
-
# Yield (key, value
|
30
|
-
def
|
31
|
-
|
30
|
+
# Yield (key, value). No locking is performed.
|
31
|
+
def self.read_all_values(f)
|
32
|
+
m = Mmap.new(f, 'rw', Mmap::MAP_SHARED)
|
33
|
+
used, = m[0..3].unpack('l')
|
34
|
+
pos = 8
|
35
|
+
values = []
|
36
|
+
while pos < used
|
37
|
+
data = m.slice(pos..-1)
|
38
|
+
encoded_len, = data.unpack('l')
|
39
|
+
value_offset = 4 + encoded_len + (8 - (encoded_len + 4) % 8)
|
40
|
+
|
41
|
+
encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset))
|
42
|
+
values << [encoded, value]
|
43
|
+
pos += value_offset + 8
|
44
|
+
end
|
45
|
+
values
|
46
|
+
ensure
|
47
|
+
m.munmap
|
32
48
|
end
|
33
49
|
|
34
50
|
def read_value(key)
|
35
51
|
@mutex.synchronize do
|
36
|
-
init_value(key) unless @positions.
|
52
|
+
init_value(key) unless @positions.key?(key)
|
37
53
|
end
|
38
54
|
pos = @positions[key]
|
39
55
|
# We assume that reading from an 8 byte aligned value is atomic.
|
40
|
-
@m[pos..pos+7].unpack('d')[0]
|
56
|
+
@m[pos..pos + 7].unpack('d')[0]
|
41
57
|
end
|
42
58
|
|
43
59
|
def write_value(key, value)
|
44
60
|
@mutex.synchronize do
|
45
|
-
init_value(key) unless @positions.
|
61
|
+
init_value(key) unless @positions.key?(key)
|
46
62
|
end
|
47
63
|
pos = @positions[key]
|
48
64
|
# We assume that writing to an 8 byte aligned value is atomic.
|
49
|
-
@m[pos..pos+7] = [value].pack('d')
|
65
|
+
@m[pos..pos + 7] = [value].pack('d')
|
50
66
|
end
|
51
67
|
|
52
68
|
def close
|
@@ -64,9 +80,7 @@ module Prometheus
|
|
64
80
|
private
|
65
81
|
|
66
82
|
def process_file
|
67
|
-
if @f.size < MINIMUM_SIZE
|
68
|
-
@f.truncate(initial_mmap_file_size)
|
69
|
-
end
|
83
|
+
@f.truncate(initial_mmap_file_size) if @f.size < MINIMUM_SIZE
|
70
84
|
|
71
85
|
@capacity = @f.size
|
72
86
|
@m = Mmap.new(@f.path, 'rw', Mmap::MAP_SHARED)
|
@@ -78,7 +92,7 @@ module Prometheus
|
|
78
92
|
@used = 8
|
79
93
|
@m[0..3] = [@used].pack('l')
|
80
94
|
else
|
81
|
-
|
95
|
+
read_all_positions.each do |key, _, pos|
|
82
96
|
@positions[key] = pos
|
83
97
|
end
|
84
98
|
end
|
@@ -104,21 +118,20 @@ module Prometheus
|
|
104
118
|
end
|
105
119
|
|
106
120
|
# Yield (key, value, pos). No locking is performed.
|
107
|
-
def
|
121
|
+
def read_all_positions
|
108
122
|
pos = 8
|
109
123
|
values = []
|
110
124
|
while pos < @used
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
pos +=
|
116
|
-
|
117
|
-
values << [encoded, value, pos]
|
125
|
+
data = @m.slice(pos..-1)
|
126
|
+
encoded_len = data.unpack('l')[0]
|
127
|
+
padding_len = 8 - (encoded_len + 4) % 8
|
128
|
+
encoded = data.unpack(format('@4A%d', encoded_len))
|
129
|
+
pos += 4 + encoded_len + padding_len
|
130
|
+
values << [encoded, pos]
|
118
131
|
pos += 8
|
119
132
|
end
|
120
133
|
values
|
121
134
|
end
|
122
135
|
end
|
123
136
|
end
|
124
|
-
end
|
137
|
+
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'prometheus/client'
|
1
2
|
require 'prometheus/client/mmaped_dict'
|
2
3
|
require 'json'
|
3
4
|
|
@@ -7,10 +8,10 @@ module Prometheus
|
|
7
8
|
class MmapedValue
|
8
9
|
@@files = {}
|
9
10
|
@@files_lock = Mutex.new
|
10
|
-
@@pid =
|
11
|
+
@@pid = -1
|
11
12
|
|
12
13
|
def initialize(type, metric_name, name, labels, multiprocess_mode = '')
|
13
|
-
@pid =
|
14
|
+
@pid = Prometheus::Client.pid
|
14
15
|
@file_prefix = type.to_s
|
15
16
|
@metric_name = metric_name
|
16
17
|
@name = name
|
@@ -50,7 +51,7 @@ module Prometheus
|
|
50
51
|
def self.reset_on_pid_change
|
51
52
|
@@files_lock.synchronize do
|
52
53
|
if pid_changed?
|
53
|
-
@@pid =
|
54
|
+
@@pid = Prometheus::Client.pid
|
54
55
|
@@files = {}
|
55
56
|
end
|
56
57
|
end
|
@@ -62,7 +63,7 @@ module Prometheus
|
|
62
63
|
end
|
63
64
|
|
64
65
|
def self.pid_changed?
|
65
|
-
@@pid !=
|
66
|
+
@@pid != Prometheus::Client.pid
|
66
67
|
end
|
67
68
|
|
68
69
|
def self.multiprocess
|
@@ -70,8 +71,8 @@ module Prometheus
|
|
70
71
|
end
|
71
72
|
|
72
73
|
def reinitialize
|
73
|
-
if @pid !=
|
74
|
-
@pid =
|
74
|
+
if @pid != Prometheus::Client.pid
|
75
|
+
@pid = Prometheus::Client.pid
|
75
76
|
initialize_file
|
76
77
|
end
|
77
78
|
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Prometheus
|
2
|
+
module Client
|
3
|
+
module Support
|
4
|
+
module Unicorn
|
5
|
+
def self.worker_pid_provider
|
6
|
+
wid = worker_id
|
7
|
+
Process.pid if wid.nil?
|
8
|
+
|
9
|
+
"worker_id_#{wid}"
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.worker_id
|
13
|
+
return unless defined?(::Unicorn::Worker)
|
14
|
+
|
15
|
+
workers = ObjectSpace.each_object(::Unicorn::Worker)
|
16
|
+
return if workers.nil?
|
17
|
+
|
18
|
+
workers_first = workers.first
|
19
|
+
workers_first.nr unless workers_first.nil?
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: prometheus-client-mmap
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.0.
|
4
|
+
version: 0.7.0.beta13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tobias Schmidt
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-08-
|
11
|
+
date: 2017-08-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mmap2
|
@@ -44,6 +44,7 @@ files:
|
|
44
44
|
- lib/prometheus/client/counter.rb
|
45
45
|
- lib/prometheus/client/formats/text.rb
|
46
46
|
- lib/prometheus/client/gauge.rb
|
47
|
+
- lib/prometheus/client/helper/json_parser.rb
|
47
48
|
- lib/prometheus/client/histogram.rb
|
48
49
|
- lib/prometheus/client/label_set_validator.rb
|
49
50
|
- lib/prometheus/client/metric.rb
|
@@ -55,6 +56,7 @@ files:
|
|
55
56
|
- lib/prometheus/client/registry.rb
|
56
57
|
- lib/prometheus/client/simple_value.rb
|
57
58
|
- lib/prometheus/client/summary.rb
|
59
|
+
- lib/prometheus/client/support/unicorn.rb
|
58
60
|
- lib/prometheus/client/uses_value_type.rb
|
59
61
|
- lib/prometheus/client/version.rb
|
60
62
|
homepage: https://gitlab.com/gitlab-org/prometheus-client-mmap
|