dogstatsd-ruby 4.0.0 → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,181 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ class Statsd
5
+ # Sender is using a companion thread to flush and pack messages
6
+ # in a `MessageBuffer`.
7
+ # The communication with this thread is done using a `Queue`.
8
+ # If the thread is dead, it is starting a new one to avoid having a blocked
9
+ # Sender with no companion thread to communicate with (most of the time, having
10
+ # a dead companion thread means that a fork just happened and that we are
11
+ # running in the child process).
12
+ class Sender
13
+ CLOSEABLE_QUEUES = Queue.instance_methods.include?(:close)
14
+
15
+ def initialize(message_buffer, telemetry: nil, queue_size: UDP_DEFAULT_BUFFER_SIZE, logger: nil, flush_interval: nil, queue_class: Queue, thread_class: Thread)
16
+ @message_buffer = message_buffer
17
+ @telemetry = telemetry
18
+ @queue_size = queue_size
19
+ @logger = logger
20
+ @mx = Mutex.new
21
+ @queue_class = queue_class
22
+ @thread_class = thread_class
23
+ @flush_timer = if flush_interval
24
+ Datadog::Statsd::Timer.new(flush_interval) { flush(sync: true) }
25
+ else
26
+ nil
27
+ end
28
+ end
29
+
30
+ def flush(sync: false)
31
+ # keep a copy around in case another thread is calling #stop while this method is running
32
+ current_message_queue = message_queue
33
+
34
+ # don't try to flush if there is no message_queue instantiated or
35
+ # no companion thread running
36
+ if !current_message_queue
37
+ @logger.debug { "Statsd: can't flush: no message queue ready" } if @logger
38
+ return
39
+ end
40
+ if !sender_thread.alive?
41
+ @logger.debug { "Statsd: can't flush: no sender_thread alive" } if @logger
42
+ return
43
+ end
44
+
45
+ current_message_queue.push(:flush)
46
+ rendez_vous if sync
47
+ end
48
+
49
+ def rendez_vous
50
+ # could happen if #start hasn't be called
51
+ return unless message_queue
52
+
53
+ # Initialize and get the thread's sync queue
54
+ queue = (@thread_class.current[:statsd_sync_queue] ||= @queue_class.new)
55
+ # tell sender-thread to notify us in the current
56
+ # thread's queue
57
+ message_queue.push(queue)
58
+ # wait for the sender thread to send a message
59
+ # once the flush is done
60
+ queue.pop
61
+ end
62
+
63
+ def add(message)
64
+ raise ArgumentError, 'Start sender first' unless message_queue
65
+
66
+ # if the thread does not exist, we assume we are running in a forked process,
67
+ # empty the message queue and message buffers (these messages belong to
68
+ # the parent process) and spawn a new companion thread.
69
+ if !sender_thread.alive?
70
+ @mx.synchronize {
71
+ # a call from another thread has already re-created
72
+ # the companion thread before this one acquired the lock
73
+ break if sender_thread.alive?
74
+ @logger.debug { "Statsd: companion thread is dead, re-creating one" } if @logger
75
+
76
+ message_queue.close if CLOSEABLE_QUEUES
77
+ @message_queue = nil
78
+ message_buffer.reset
79
+ start
80
+ @flush_timer.start if @flush_timer && @flush_timer.stop?
81
+ }
82
+ end
83
+
84
+ if message_queue.length <= @queue_size
85
+ message_queue << message
86
+ else
87
+ @telemetry.dropped_queue(packets: 1, bytes: message.bytesize) if @telemetry
88
+ end
89
+ end
90
+
91
+ def start
92
+ raise ArgumentError, 'Sender already started' if message_queue
93
+
94
+ # initialize a new message queue for the background thread
95
+ @message_queue = @queue_class.new
96
+ # start background thread
97
+ @sender_thread = @thread_class.new(&method(:send_loop))
98
+ @sender_thread.name = "Statsd Sender" unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
99
+ @flush_timer.start if @flush_timer
100
+ end
101
+
102
+ if CLOSEABLE_QUEUES
103
+ # when calling stop, make sure that no other threads is trying
104
+ # to close the sender nor trying to continue to `#add` more message
105
+ # into the sender.
106
+ def stop(join_worker: true)
107
+ @flush_timer.stop if @flush_timer
108
+
109
+ message_queue = @message_queue
110
+ message_queue.close if message_queue
111
+
112
+ sender_thread = @sender_thread
113
+ sender_thread.join if sender_thread && join_worker
114
+ end
115
+ else
116
+ # when calling stop, make sure that no other threads is trying
117
+ # to close the sender nor trying to continue to `#add` more message
118
+ # into the sender.
119
+ def stop(join_worker: true)
120
+ @flush_timer.stop if @flush_timer
121
+
122
+ message_queue = @message_queue
123
+ message_queue << :close if message_queue
124
+
125
+ sender_thread = @sender_thread
126
+ sender_thread.join if sender_thread && join_worker
127
+ end
128
+ end
129
+
130
+ private
131
+
132
+ attr_reader :message_buffer
133
+ attr_reader :message_queue
134
+ attr_reader :sender_thread
135
+
136
+ if CLOSEABLE_QUEUES
137
+ def send_loop
138
+ until (message = message_queue.pop).nil? && message_queue.closed?
139
+ # skip if message is nil, e.g. when message_queue
140
+ # is empty and closed
141
+ next unless message
142
+
143
+ case message
144
+ when :flush
145
+ message_buffer.flush
146
+ when @queue_class
147
+ message.push(:go_on)
148
+ else
149
+ message_buffer.add(message)
150
+ end
151
+ end
152
+
153
+ @message_queue = nil
154
+ @sender_thread = nil
155
+ end
156
+ else
157
+ def send_loop
158
+ loop do
159
+ message = message_queue.pop
160
+
161
+ next unless message
162
+
163
+ case message
164
+ when :close
165
+ break
166
+ when :flush
167
+ message_buffer.flush
168
+ when @queue_class
169
+ message.push(:go_on)
170
+ else
171
+ message_buffer.add(message)
172
+ end
173
+ end
174
+
175
+ @message_queue = nil
176
+ @sender_thread = nil
177
+ end
178
+ end
179
+ end
180
+ end
181
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ class Statsd
5
+ module Serialization
6
+ class EventSerializer
7
+ EVENT_BASIC_OPTIONS = {
8
+ date_happened: 'd:',
9
+ hostname: 'h:',
10
+ aggregation_key: 'k:',
11
+ priority: 'p:',
12
+ source_type_name: 's:',
13
+ alert_type: 't:',
14
+ }.freeze
15
+
16
+ def initialize(global_tags: [])
17
+ @tag_serializer = TagSerializer.new(global_tags)
18
+ end
19
+
20
+ def format(title, text, options = EMPTY_OPTIONS)
21
+ title = escape(title)
22
+ text = escape(text)
23
+
24
+ String.new.tap do |event|
25
+ event << '_e{'
26
+ event << title.bytesize.to_s
27
+ event << ','
28
+ event << text.bytesize.to_s
29
+ event << '}:'
30
+ event << title
31
+ event << '|'
32
+ event << text
33
+
34
+ # we are serializing the generic service check options
35
+ # before serializing specialized options that need edge-cases
36
+ EVENT_BASIC_OPTIONS.each do |option_key, shortcut|
37
+ if value = options[option_key]
38
+ event << '|'
39
+ event << shortcut
40
+ event << value.to_s.delete('|')
41
+ end
42
+ end
43
+
44
+ # also returns the global tags from serializer
45
+ if tags = tag_serializer.format(options[:tags])
46
+ event << '|#'
47
+ event << tags
48
+ end
49
+
50
+ if event.bytesize > MAX_EVENT_SIZE
51
+ if options[:truncate_if_too_long]
52
+ event.slice!(MAX_EVENT_SIZE..event.length)
53
+ else
54
+ raise "Event #{title} payload is too big (more that 8KB), event discarded"
55
+ end
56
+ end
57
+ end
58
+ end
59
+
60
+ protected
61
+ attr_reader :tag_serializer
62
+
63
+ def escape(text)
64
+ text.delete('|').tap do |t|
65
+ t.gsub!("\n", '\n')
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ # require 'forwardable'
4
+
5
+ module Datadog
6
+ class Statsd
7
+ module Serialization
8
+ class Serializer
9
+ def initialize(prefix: nil, global_tags: [])
10
+ @stat_serializer = StatSerializer.new(prefix, global_tags: global_tags)
11
+ @service_check_serializer = ServiceCheckSerializer.new(global_tags: global_tags)
12
+ @event_serializer = EventSerializer.new(global_tags: global_tags)
13
+ end
14
+
15
+ # using *args would make new allocations
16
+ def to_stat(name, delta, type, tags: [], sample_rate: 1)
17
+ stat_serializer.format(name, delta, type, tags: tags, sample_rate: sample_rate)
18
+ end
19
+
20
+ # using *args would make new allocations
21
+ def to_service_check(name, status, options = EMPTY_OPTIONS)
22
+ service_check_serializer.format(name, status, options)
23
+ end
24
+
25
+ # using *args would make new allocations
26
+ def to_event(title, text, options = EMPTY_OPTIONS)
27
+ event_serializer.format(title, text, options)
28
+ end
29
+
30
+ def global_tags
31
+ stat_serializer.global_tags
32
+ end
33
+
34
+ protected
35
+ attr_reader :stat_serializer
36
+ attr_reader :service_check_serializer
37
+ attr_reader :event_serializer
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ class Statsd
5
+ module Serialization
6
+ class ServiceCheckSerializer
7
+ SERVICE_CHECK_BASIC_OPTIONS = {
8
+ timestamp: 'd:',
9
+ hostname: 'h:',
10
+ }.freeze
11
+
12
+ def initialize(global_tags: [])
13
+ @tag_serializer = TagSerializer.new(global_tags)
14
+ end
15
+
16
+ def format(name, status, options = EMPTY_OPTIONS)
17
+ String.new.tap do |service_check|
18
+ # line basics
19
+ service_check << "_sc"
20
+ service_check << "|"
21
+ service_check << name.to_s
22
+ service_check << "|"
23
+ service_check << status.to_s
24
+
25
+ # we are serializing the generic service check options
26
+ # before serializing specialized options that need edge-cases
27
+ SERVICE_CHECK_BASIC_OPTIONS.each do |option_key, shortcut|
28
+ if value = options[option_key]
29
+ service_check << '|'
30
+ service_check << shortcut
31
+ service_check << value.to_s.delete('|')
32
+ end
33
+ end
34
+
35
+ if message = options[:message]
36
+ service_check << '|m:'
37
+ service_check << escape_message(message)
38
+ end
39
+
40
+ # also returns the global tags from serializer
41
+ if tags = tag_serializer.format(options[:tags])
42
+ service_check << '|#'
43
+ service_check << tags
44
+ end
45
+ end
46
+ end
47
+
48
+ protected
49
+ attr_reader :tag_serializer
50
+
51
+ def escape_message(message)
52
+ message.delete('|').tap do |m|
53
+ m.gsub!("\n", '\n')
54
+ m.gsub!('m:', 'm\:')
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ class Statsd
5
+ module Serialization
6
+ class StatSerializer
7
+ def initialize(prefix, global_tags: [])
8
+ @prefix = prefix
9
+ @prefix_str = prefix.to_s
10
+ @tag_serializer = TagSerializer.new(global_tags)
11
+ end
12
+
13
+ def format(name, delta, type, tags: [], sample_rate: 1)
14
+ name = formated_name(name)
15
+
16
+ if sample_rate != 1
17
+ if tags_list = tag_serializer.format(tags)
18
+ "#{@prefix_str}#{name}:#{delta}|#{type}|@#{sample_rate}|##{tags_list}"
19
+ else
20
+ "#{@prefix_str}#{name}:#{delta}|#{type}|@#{sample_rate}"
21
+ end
22
+ else
23
+ if tags_list = tag_serializer.format(tags)
24
+ "#{@prefix_str}#{name}:#{delta}|#{type}|##{tags_list}"
25
+ else
26
+ "#{@prefix_str}#{name}:#{delta}|#{type}"
27
+ end
28
+ end
29
+ end
30
+
31
+ def global_tags
32
+ tag_serializer.global_tags
33
+ end
34
+
35
+ private
36
+
37
+ attr_reader :prefix
38
+ attr_reader :tag_serializer
39
+
40
+ def formated_name(name)
41
+ if name.is_a?(String)
42
+ # DEV: gsub is faster than dup.gsub!
43
+ formated = name.gsub('::', '.')
44
+ else
45
+ formated = name.to_s
46
+ formated.gsub!('::', '.')
47
+ end
48
+
49
+ formated.tr!(':|@', '_')
50
+ formated
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ class Statsd
5
+ module Serialization
6
+ class TagSerializer
7
+ def initialize(global_tags = [], env = ENV)
8
+ # Convert to hash
9
+ global_tags = to_tags_hash(global_tags)
10
+
11
+ # Merge with default tags
12
+ global_tags = default_tags(env).merge(global_tags)
13
+
14
+ # Convert to tag list and set
15
+ @global_tags = to_tags_list(global_tags)
16
+ if @global_tags.any?
17
+ @global_tags_formatted = @global_tags.join(',')
18
+ else
19
+ @global_tags_formatted = nil
20
+ end
21
+ end
22
+
23
+ def format(message_tags)
24
+ if !message_tags || message_tags.empty?
25
+ return @global_tags_formatted
26
+ end
27
+
28
+ tags = if @global_tags_formatted
29
+ [@global_tags_formatted, to_tags_list(message_tags)]
30
+ else
31
+ to_tags_list(message_tags)
32
+ end
33
+
34
+ tags.join(',')
35
+ end
36
+
37
+ attr_reader :global_tags
38
+
39
+ private
40
+
41
+ def to_tags_hash(tags)
42
+ case tags
43
+ when Hash
44
+ tags.dup
45
+ when Array
46
+ Hash[
47
+ tags.map do |string|
48
+ tokens = string.split(':')
49
+ tokens << nil if tokens.length == 1
50
+ tokens.length == 2 ? tokens : nil
51
+ end.compact
52
+ ]
53
+ else
54
+ {}
55
+ end
56
+ end
57
+
58
+ def to_tags_list(tags)
59
+ case tags
60
+ when Hash
61
+ tags.map do |name, value|
62
+ if value
63
+ escape_tag_content("#{name}:#{value}")
64
+ else
65
+ escape_tag_content(name)
66
+ end
67
+ end
68
+ when Array
69
+ tags.map { |tag| escape_tag_content(tag) }
70
+ else
71
+ []
72
+ end
73
+ end
74
+
75
+ def escape_tag_content(tag)
76
+ tag.to_s.delete('|,')
77
+ end
78
+
79
+ def dd_tags(env = ENV)
80
+ return {} unless dd_tags = env['DD_TAGS']
81
+
82
+ to_tags_hash(dd_tags.split(','))
83
+ end
84
+
85
+ def default_tags(env = ENV)
86
+ dd_tags(env).tap do |tags|
87
+ tags['dd.internal.entity_id'] = env['DD_ENTITY_ID'] if env.key?('DD_ENTITY_ID')
88
+ tags['env'] = env['DD_ENV'] if env.key?('DD_ENV')
89
+ tags['service'] = env['DD_SERVICE'] if env.key?('DD_SERVICE')
90
+ tags['version'] = env['DD_VERSION'] if env.key?('DD_VERSION')
91
+ end
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ class Statsd
5
+ module Serialization
6
+ end
7
+ end
8
+ end
9
+
10
+ require_relative 'serialization/tag_serializer'
11
+ require_relative 'serialization/service_check_serializer'
12
+ require_relative 'serialization/event_serializer'
13
+ require_relative 'serialization/stat_serializer'
14
+
15
+ require_relative 'serialization/serializer'
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ class Statsd
5
+ # The SingleThreadSender is a sender synchronously buffering messages
6
+ # in a `MessageBuffer`.
7
+ # It is using current Process.PID to check it is the result of a recent fork
8
+ # and it is reseting the MessageBuffer if that's the case.
9
+ class SingleThreadSender
10
+ def initialize(message_buffer, logger: nil, flush_interval: nil)
11
+ @message_buffer = message_buffer
12
+ @logger = logger
13
+ @mx = Mutex.new
14
+ @flush_timer = if flush_interval
15
+ Datadog::Statsd::Timer.new(flush_interval) { flush }
16
+ else
17
+ nil
18
+ end
19
+ # store the pid for which this sender has been created
20
+ update_fork_pid
21
+ end
22
+
23
+ def add(message)
24
+ @mx.synchronize {
25
+ # we have just forked, meaning we have messages in the buffer that we should
26
+ # not send, they belong to the parent process, let's clear the buffer.
27
+ if forked?
28
+ @message_buffer.reset
29
+ @flush_timer.start if @flush_timer && @flush_timer.stop?
30
+ update_fork_pid
31
+ end
32
+ @message_buffer.add(message)
33
+ }
34
+ end
35
+
36
+ def flush(*)
37
+ @mx.synchronize {
38
+ @message_buffer.flush()
39
+ }
40
+ end
41
+
42
+ def start()
43
+ @flush_timer.start if @flush_timer
44
+ end
45
+
46
+ def stop()
47
+ @flush_timer.stop if @flush_timer
48
+ end
49
+
50
+ # Compatibility with `Sender`
51
+ def rendez_vous()
52
+ end
53
+
54
+ private
55
+
56
+ # below are "fork management" methods to be able to clean the MessageBuffer
57
+ # if it detects that it is running in a unknown PID.
58
+
59
+ def forked?
60
+ Process.pid != @fork_pid
61
+ end
62
+
63
+ def update_fork_pid
64
+ @fork_pid = Process.pid
65
+ end
66
+ end
67
+ end
68
+ end