vinted-prometheus-client-mmap 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +2 -0
  3. data/ext/fast_mmaped_file_rs/Cargo.toml +40 -0
  4. data/ext/fast_mmaped_file_rs/README.md +52 -0
  5. data/ext/fast_mmaped_file_rs/build.rs +7 -0
  6. data/ext/fast_mmaped_file_rs/extconf.rb +28 -0
  7. data/ext/fast_mmaped_file_rs/src/error.rs +174 -0
  8. data/ext/fast_mmaped_file_rs/src/exemplars.rs +25 -0
  9. data/ext/fast_mmaped_file_rs/src/file_entry.rs +1190 -0
  10. data/ext/fast_mmaped_file_rs/src/file_info.rs +240 -0
  11. data/ext/fast_mmaped_file_rs/src/lib.rs +87 -0
  12. data/ext/fast_mmaped_file_rs/src/macros.rs +14 -0
  13. data/ext/fast_mmaped_file_rs/src/map.rs +492 -0
  14. data/ext/fast_mmaped_file_rs/src/metrics.proto +153 -0
  15. data/ext/fast_mmaped_file_rs/src/mmap/inner.rs +704 -0
  16. data/ext/fast_mmaped_file_rs/src/mmap.rs +896 -0
  17. data/ext/fast_mmaped_file_rs/src/raw_entry.rs +473 -0
  18. data/ext/fast_mmaped_file_rs/src/testhelper.rs +222 -0
  19. data/ext/fast_mmaped_file_rs/src/util.rs +121 -0
  20. data/lib/.DS_Store +0 -0
  21. data/lib/prometheus/.DS_Store +0 -0
  22. data/lib/prometheus/client/configuration.rb +23 -0
  23. data/lib/prometheus/client/counter.rb +27 -0
  24. data/lib/prometheus/client/formats/protobuf.rb +92 -0
  25. data/lib/prometheus/client/formats/text.rb +85 -0
  26. data/lib/prometheus/client/gauge.rb +40 -0
  27. data/lib/prometheus/client/helper/entry_parser.rb +132 -0
  28. data/lib/prometheus/client/helper/file_locker.rb +50 -0
  29. data/lib/prometheus/client/helper/json_parser.rb +23 -0
  30. data/lib/prometheus/client/helper/metrics_processing.rb +45 -0
  31. data/lib/prometheus/client/helper/metrics_representation.rb +51 -0
  32. data/lib/prometheus/client/helper/mmaped_file.rb +64 -0
  33. data/lib/prometheus/client/helper/plain_file.rb +29 -0
  34. data/lib/prometheus/client/histogram.rb +80 -0
  35. data/lib/prometheus/client/label_set_validator.rb +85 -0
  36. data/lib/prometheus/client/metric.rb +80 -0
  37. data/lib/prometheus/client/mmaped_dict.rb +79 -0
  38. data/lib/prometheus/client/mmaped_value.rb +158 -0
  39. data/lib/prometheus/client/page_size.rb +17 -0
  40. data/lib/prometheus/client/push.rb +203 -0
  41. data/lib/prometheus/client/rack/collector.rb +88 -0
  42. data/lib/prometheus/client/rack/exporter.rb +102 -0
  43. data/lib/prometheus/client/registry.rb +65 -0
  44. data/lib/prometheus/client/simple_value.rb +31 -0
  45. data/lib/prometheus/client/summary.rb +69 -0
  46. data/lib/prometheus/client/support/puma.rb +44 -0
  47. data/lib/prometheus/client/support/unicorn.rb +35 -0
  48. data/lib/prometheus/client/uses_value_type.rb +20 -0
  49. data/lib/prometheus/client/version.rb +5 -0
  50. data/lib/prometheus/client.rb +58 -0
  51. data/lib/prometheus.rb +3 -0
  52. metadata +203 -0
@@ -0,0 +1,51 @@
1
+ module Prometheus
2
+ module Client
3
+ module Helper
4
+ module MetricsRepresentation
5
+ METRIC_LINE = '%s%s %s'.freeze
6
+ TYPE_LINE = '# TYPE %s %s'.freeze
7
+ HELP_LINE = '# HELP %s %s'.freeze
8
+
9
+ LABEL = '%s="%s"'.freeze
10
+ SEPARATOR = ','.freeze
11
+ DELIMITER = "\n".freeze
12
+
13
+ REGEX = { doc: /[\n\\]/, label: /[\n\\"]/ }.freeze
14
+ REPLACE = { "\n" => '\n', '\\' => '\\\\', '"' => '\"' }.freeze
15
+
16
+ def self.to_text(metrics)
17
+ lines = []
18
+
19
+ metrics.each do |name, metric|
20
+ lines << format(HELP_LINE, name, escape(metric[:help]))
21
+ lines << format(TYPE_LINE, name, metric[:type])
22
+ metric[:samples].each do |metric_name, labels, value|
23
+ lines << metric(metric_name, format_labels(labels), value)
24
+ end
25
+ end
26
+
27
+ # there must be a trailing delimiter
28
+ (lines << nil).join(DELIMITER)
29
+ end
30
+
31
+ def self.metric(name, labels, value)
32
+ format(METRIC_LINE, name, labels, value)
33
+ end
34
+
35
+ def self.format_labels(set)
36
+ return if set.empty?
37
+
38
+ strings = set.each_with_object([]) do |(key, value), memo|
39
+ memo << format(LABEL, key, escape(value, :label))
40
+ end
41
+
42
+ "{#{strings.join(SEPARATOR)}}"
43
+ end
44
+
45
+ def self.escape(string, format = :doc)
46
+ string.to_s.gsub(REGEX[format], REPLACE)
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,64 @@
1
+ require 'prometheus/client/helper/entry_parser'
2
+ require 'prometheus/client/helper/file_locker'
3
+
4
+ # load precompiled extension if available
5
+ begin
6
+ ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
7
+ require_relative "../../../#{ruby_version}/fast_mmaped_file_rs"
8
+ rescue LoadError
9
+ require 'fast_mmaped_file_rs'
10
+ end
11
+
12
+ module Prometheus
13
+ module Client
14
+ module Helper
15
+ class MmapedFile < FastMmapedFileRs
16
+ include EntryParser
17
+
18
+ attr_reader :filepath, :size
19
+
20
+ def initialize(filepath)
21
+ @filepath = filepath
22
+
23
+ File.open(filepath, 'a+b') do |file|
24
+ file.truncate(initial_mmap_file_size) if file.size < MINIMUM_SIZE
25
+ @size = file.size
26
+ end
27
+
28
+ super(filepath)
29
+ end
30
+
31
+ def close
32
+ munmap
33
+ FileLocker.unlock(filepath)
34
+ end
35
+
36
+ private
37
+
38
+ def initial_mmap_file_size
39
+ Prometheus::Client.configuration.initial_mmap_file_size
40
+ end
41
+
42
+ public
43
+
44
+ class << self
45
+ def open(filepath)
46
+ MmapedFile.new(filepath)
47
+ end
48
+
49
+ def ensure_exclusive_file(file_prefix = 'mmaped_file')
50
+ (0..Float::INFINITY).lazy
51
+ .map { |f_num| "#{file_prefix}_#{Prometheus::Client.pid}-#{f_num}.db" }
52
+ .map { |filename| File.join(Prometheus::Client.configuration.multiprocess_files_dir, filename) }
53
+ .find { |path| Helper::FileLocker.lock_to_process(path) }
54
+ end
55
+
56
+ def open_exclusive_file(file_prefix = 'mmaped_file')
57
+ filename = Helper::MmapedFile.ensure_exclusive_file(file_prefix)
58
+ open(filename)
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,29 @@
1
+ require 'prometheus/client/helper/entry_parser'
2
+
3
+ module Prometheus
4
+ module Client
5
+ module Helper
6
+ # Parses DB files without using mmap
7
+ class PlainFile
8
+ include EntryParser
9
+ attr_reader :filepath
10
+
11
+ def source
12
+ @data ||= File.read(filepath, mode: 'rb')
13
+ end
14
+
15
+ def initialize(filepath)
16
+ @filepath = filepath
17
+ end
18
+
19
+ def slice(*args)
20
+ source.slice(*args)
21
+ end
22
+
23
+ def size
24
+ source.length
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,80 @@
1
+ require 'prometheus/client/metric'
2
+ require 'prometheus/client/uses_value_type'
3
+
4
+ module Prometheus
5
+ module Client
6
+ # A histogram samples observations (usually things like request durations
7
+ # or response sizes) and counts them in configurable buckets. It also
8
+ # provides a sum of all observed values.
9
+ class Histogram < Metric
10
+ # Value represents the state of a Histogram at a given point.
11
+ class Value < Hash
12
+ include UsesValueType
13
+ attr_accessor :sum, :total, :total_inf
14
+
15
+ def initialize(type, name, labels, buckets)
16
+ @sum = value_object(type, name, "#{name}_sum", labels)
17
+ @total = value_object(type, name, "#{name}_count", labels)
18
+ @total_inf = value_object(type, name, "#{name}_bucket", labels.merge(le: "+Inf"))
19
+
20
+ buckets.each do |bucket|
21
+ self[bucket] = value_object(type, name, "#{name}_bucket", labels.merge(le: bucket.to_s))
22
+ end
23
+ end
24
+
25
+ def observe(value, exemplar_name = '', exemplar_value = '')
26
+ @sum.increment(value, exemplar_name, exemplar_value)
27
+ @total.increment(1, exemplar_name, exemplar_value)
28
+ @total_inf.increment(1, exemplar_name, exemplar_value)
29
+
30
+ each_key do |bucket|
31
+ self[bucket].increment(1, exemplar_name, exemplar_value) if value <= bucket
32
+ end
33
+ end
34
+
35
+ def get()
36
+ hash = {}
37
+ each_key do |bucket|
38
+ hash[bucket] = self[bucket].get()
39
+ end
40
+ hash
41
+ end
42
+ end
43
+
44
+ # DEFAULT_BUCKETS are the default Histogram buckets. The default buckets
45
+ # are tailored to broadly measure the response time (in seconds) of a
46
+ # network service. (From DefBuckets client_golang)
47
+ DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1,
48
+ 2.5, 5, 10].freeze
49
+
50
+ # Offer a way to manually specify buckets
51
+ def initialize(name, docstring, base_labels = {},
52
+ buckets = DEFAULT_BUCKETS)
53
+ raise ArgumentError, 'Unsorted buckets, typo?' unless sorted? buckets
54
+
55
+ @buckets = buckets
56
+ super(name, docstring, base_labels)
57
+ end
58
+
59
+ def type
60
+ :histogram
61
+ end
62
+
63
+ def observe(labels, value)
64
+ label_set = label_set_for(labels)
65
+ synchronize { @values[label_set].observe(value) }
66
+ end
67
+
68
+ private
69
+
70
+ def default(labels)
71
+ # TODO: default function needs to know key of hash info (label names and values)
72
+ Value.new(type, @name, labels, @buckets)
73
+ end
74
+
75
+ def sorted?(bucket)
76
+ bucket.each_cons(2).all? { |i, j| i <= j }
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,85 @@
1
+ # encoding: UTF-8
2
+
3
+ module Prometheus
4
+ module Client
5
+ # LabelSetValidator ensures that all used label sets comply with the
6
+ # Prometheus specification.
7
+ class LabelSetValidator
8
+ RESERVED_LABELS = [].freeze
9
+
10
+ class LabelSetError < StandardError; end
11
+ class InvalidLabelSetError < LabelSetError; end
12
+ class InvalidLabelError < LabelSetError; end
13
+ class ReservedLabelError < LabelSetError; end
14
+
15
+ def initialize(reserved_labels = [])
16
+ @reserved_labels = (reserved_labels + RESERVED_LABELS).freeze
17
+ @validated = {}
18
+ end
19
+
20
+ def valid?(labels)
21
+ unless labels.is_a?(Hash)
22
+ raise InvalidLabelSetError, "#{labels} is not a valid label set"
23
+ end
24
+
25
+ labels.all? do |key, value|
26
+ validate_symbol(key)
27
+ validate_name(key)
28
+ validate_reserved_key(key)
29
+ validate_value(key, value)
30
+ end
31
+ end
32
+
33
+ def validate(labels)
34
+ return labels if @validated.key?(labels.hash)
35
+
36
+ valid?(labels)
37
+
38
+ unless @validated.empty? || match?(labels, @validated.first.last)
39
+ raise InvalidLabelSetError, "labels must have the same signature: (#{label_diff(labels, @validated.first.last)})"
40
+ end
41
+
42
+ @validated[labels.hash] = labels
43
+ end
44
+
45
+ private
46
+
47
+ def label_diff(a, b)
48
+ "expected keys: #{b.keys.sort}, got: #{a.keys.sort}"
49
+ end
50
+
51
+ def match?(a, b)
52
+ a.keys.sort == b.keys.sort
53
+ end
54
+
55
+ def validate_symbol(key)
56
+ return true if key.is_a?(Symbol)
57
+
58
+ raise InvalidLabelError, "label #{key} is not a symbol"
59
+ end
60
+
61
+ def validate_name(key)
62
+ return true unless key.to_s.start_with?('__')
63
+
64
+ raise ReservedLabelError, "label #{key} must not start with __"
65
+ end
66
+
67
+ def validate_reserved_key(key)
68
+ return true unless @reserved_labels.include?(key)
69
+
70
+ raise ReservedLabelError, "#{key} is reserved"
71
+ end
72
+
73
+ def validate_value(key, value)
74
+ return true if value.is_a?(String) ||
75
+ value.is_a?(Numeric) ||
76
+ value.is_a?(Symbol) ||
77
+ value.is_a?(FalseClass) ||
78
+ value.is_a?(TrueClass) ||
79
+ value.nil?
80
+
81
+ raise InvalidLabelError, "#{key} does not contain a valid value (type #{value.class})"
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,80 @@
1
+ require 'thread'
2
+ require 'prometheus/client/label_set_validator'
3
+ require 'prometheus/client/uses_value_type'
4
+
5
+ module Prometheus
6
+ module Client
7
+ class Metric
8
+ include UsesValueType
9
+ attr_reader :name, :docstring, :base_labels
10
+
11
+ def initialize(name, docstring, base_labels = {})
12
+ @mutex = Mutex.new
13
+ @validator = case type
14
+ when :summary
15
+ LabelSetValidator.new(['quantile'])
16
+ when :histogram
17
+ LabelSetValidator.new(['le'])
18
+ else
19
+ LabelSetValidator.new
20
+ end
21
+ @values = Hash.new { |hash, key| hash[key] = default(key) }
22
+
23
+ validate_name(name)
24
+ validate_docstring(docstring)
25
+ @validator.valid?(base_labels)
26
+
27
+ @name = name
28
+ @docstring = docstring
29
+ @base_labels = base_labels
30
+ end
31
+
32
+ # Returns the value for the given label set
33
+ def get(labels = {})
34
+ label_set = label_set_for(labels)
35
+ @validator.valid?(label_set)
36
+
37
+ @values[label_set].get
38
+ end
39
+
40
+ # Returns all label sets with their values
41
+ def values
42
+ synchronize do
43
+ @values.each_with_object({}) do |(labels, value), memo|
44
+ memo[labels] = value
45
+ end
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ def touch_default_value
52
+ @values[label_set_for({})]
53
+ end
54
+
55
+ def default(labels)
56
+ value_object(type, @name, @name, labels)
57
+ end
58
+
59
+ def validate_name(name)
60
+ return true if name.is_a?(Symbol)
61
+
62
+ raise ArgumentError, 'given name must be a symbol'
63
+ end
64
+
65
+ def validate_docstring(docstring)
66
+ return true if docstring.respond_to?(:empty?) && !docstring.empty?
67
+
68
+ raise ArgumentError, 'docstring must be given'
69
+ end
70
+
71
+ def label_set_for(labels)
72
+ @validator.validate(@base_labels.merge(labels))
73
+ end
74
+
75
+ def synchronize(&block)
76
+ @mutex.synchronize(&block)
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,79 @@
1
+ require 'prometheus/client/helper/mmaped_file'
2
+ require 'prometheus/client/helper/plain_file'
3
+ require 'prometheus/client'
4
+
5
+ module Prometheus
6
+ module Client
7
+ class ParsingError < StandardError
8
+ end
9
+
10
+ # A dict of doubles, backed by an mmapped file.
11
+ #
12
+ # The file starts with a 4 byte int, indicating how much of it is used.
13
+ # Then 4 bytes of padding.
14
+ # There's then a number of entries, consisting of a 4 byte int which is the
15
+ # size of the next field, a utf-8 encoded string key, padding to an 8 byte
16
+ # alignment, and then a 8 byte float which is the value.
17
+ class MmapedDict
18
+ attr_reader :m, :used, :positions
19
+
20
+ def self.read_all_values(f)
21
+ Helper::PlainFile.new(f).entries.map do |data, encoded_len, value_offset, _|
22
+ encoded, value = data.unpack(format('@4A%d@%dd', encoded_len, value_offset))
23
+ [encoded, value]
24
+ end
25
+ end
26
+
27
+ def initialize(m)
28
+ @mutex = Mutex.new
29
+
30
+ @m = m
31
+ # @m.mlock # TODO: Ensure memory is locked to RAM
32
+
33
+ @positions = {}
34
+ read_all_positions.each do |key, pos|
35
+ @positions[key] = pos
36
+ end
37
+ rescue StandardError => e
38
+ raise ParsingError, "exception #{e} while processing metrics file #{path}"
39
+ end
40
+
41
+ def read_value(key)
42
+ @m.fetch_entry(@positions, key, 0.0)
43
+ end
44
+
45
+ def write_value(key, value)
46
+ @m.upsert_entry(@positions, key, value)
47
+ end
48
+
49
+ def path
50
+ @m.filepath if @m
51
+ end
52
+
53
+ def close
54
+ @m.sync
55
+ @m.close
56
+ rescue TypeError => e
57
+ Prometheus::Client.logger.warn("munmap raised error #{e}")
58
+ end
59
+
60
+ def inspect
61
+ "#<#{self.class}:0x#{(object_id << 1).to_s(16)}>"
62
+ end
63
+
64
+ private
65
+
66
+ def init_value(key)
67
+ @m.add_entry(@positions, key, 0.0)
68
+ end
69
+
70
+ # Yield (key, pos). No locking is performed.
71
+ def read_all_positions
72
+ @m.entries.map do |data, encoded_len, _, absolute_pos|
73
+ encoded, = data.unpack(format('@4A%d', encoded_len))
74
+ [encoded, absolute_pos]
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,158 @@
1
+ require 'prometheus/client'
2
+ require 'prometheus/client/mmaped_dict'
3
+ require 'json'
4
+
5
+ module Prometheus
6
+ module Client
7
+ # A float protected by a mutex backed by a per-process mmaped file.
8
+ class MmapedValue
9
+ VALUE_LOCK = Mutex.new
10
+
11
+ @@files = {}
12
+ @@pid = -1
13
+
14
+ def initialize(type, metric_name, name, labels, multiprocess_mode = '')
15
+ @file_prefix = type.to_s
16
+ @metric_name = metric_name
17
+ @name = name
18
+ @labels = labels
19
+ if type == :gauge
20
+ @file_prefix += '_' + multiprocess_mode.to_s
21
+ end
22
+
23
+ @pid = -1
24
+
25
+ @mutex = Mutex.new
26
+ initialize_file
27
+ end
28
+
29
+ def increment(amount = 1, exemplar_name = '', exemplar_value = '')
30
+ @mutex.synchronize do
31
+ initialize_file if pid_changed?
32
+
33
+ @value += amount
34
+ # TODO(GiedriusS): write exemplars too.
35
+ if @file_prefix != 'gauge'
36
+ puts "#{@name} exemplar name = #{exemplar_name}, exemplar_value = #{exemplar_value}"
37
+ end
38
+ write_value(@key, @value)
39
+ @value
40
+ end
41
+ end
42
+
43
+ def decrement(amount = 1)
44
+ increment(-amount)
45
+ end
46
+
47
+ def set(value)
48
+ @mutex.synchronize do
49
+ initialize_file if pid_changed?
50
+
51
+ @value = value
52
+ write_value(@key, @value)
53
+ @value
54
+ end
55
+ end
56
+
57
+ def get
58
+ @mutex.synchronize do
59
+ initialize_file if pid_changed?
60
+ return @value
61
+ end
62
+ end
63
+
64
+ def pid_changed?
65
+ @pid != Process.pid
66
+ end
67
+
68
+ # method needs to be run in VALUE_LOCK mutex
69
+ def unsafe_reinitialize_file(check_pid = true)
70
+ unsafe_initialize_file if !check_pid || pid_changed?
71
+ end
72
+
73
+ def self.reset_and_reinitialize
74
+ VALUE_LOCK.synchronize do
75
+ @@pid = Process.pid
76
+ @@files = {}
77
+
78
+ ObjectSpace.each_object(MmapedValue).each do |v|
79
+ v.unsafe_reinitialize_file(false)
80
+ end
81
+ end
82
+ end
83
+
84
+ def self.reset_on_pid_change
85
+ if pid_changed?
86
+ @@pid = Process.pid
87
+ @@files = {}
88
+ end
89
+ end
90
+
91
+ def self.reinitialize_on_pid_change
92
+ VALUE_LOCK.synchronize do
93
+ reset_on_pid_change
94
+
95
+ ObjectSpace.each_object(MmapedValue, &:unsafe_reinitialize_file)
96
+ end
97
+ end
98
+
99
+ def self.pid_changed?
100
+ @@pid != Process.pid
101
+ end
102
+
103
+ def self.multiprocess
104
+ true
105
+ end
106
+
107
+ private
108
+
109
+ def initialize_file
110
+ VALUE_LOCK.synchronize do
111
+ unsafe_initialize_file
112
+ end
113
+ end
114
+
115
+ def unsafe_initialize_file
116
+ self.class.reset_on_pid_change
117
+
118
+ @pid = Process.pid
119
+ unless @@files.has_key?(@file_prefix)
120
+ unless @file.nil?
121
+ @file.close
122
+ end
123
+ mmaped_file = Helper::MmapedFile.open_exclusive_file(@file_prefix)
124
+
125
+ @@files[@file_prefix] = MmapedDict.new(mmaped_file)
126
+ end
127
+
128
+ @file = @@files[@file_prefix]
129
+ @key = rebuild_key
130
+
131
+ @value = read_value(@key)
132
+ end
133
+
134
+
135
+ def rebuild_key
136
+ keys = @labels.keys.sort
137
+ values = @labels.values_at(*keys)
138
+
139
+ [@metric_name, @name, keys, values].to_json
140
+ end
141
+
142
+ def write_value(key, val)
143
+ @file.write_value(key, val)
144
+ rescue StandardError => e
145
+ Prometheus::Client.logger.warn("writing value to #{@file.path} failed with #{e}")
146
+ Prometheus::Client.logger.debug(e.backtrace.join("\n"))
147
+ end
148
+
149
+ def read_value(key)
150
+ @file.read_value(key)
151
+ rescue StandardError => e
152
+ Prometheus::Client.logger.warn("reading value from #{@file.path} failed with #{e}")
153
+ Prometheus::Client.logger.debug(e.backtrace.join("\n"))
154
+ 0
155
+ end
156
+ end
157
+ end
158
+ end
@@ -0,0 +1,17 @@
1
+ require 'open3'
2
+
3
+ module Prometheus
4
+ module Client
5
+ module PageSize
6
+ def self.page_size(fallback_page_size: 4096)
7
+ stdout, status = Open3.capture2('getconf PAGESIZE')
8
+ return fallback_page_size if status.nil? || !status.success?
9
+
10
+ page_size = stdout.chomp.to_i
11
+ return fallback_page_size if page_size <= 0
12
+
13
+ page_size
14
+ end
15
+ end
16
+ end
17
+ end