statsd-instrument 3.7.0 → 3.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/benchmark.yml +0 -1
- data/.github/workflows/lint.yml +0 -1
- data/.github/workflows/tests.yml +1 -1
- data/.rubocop.yml +0 -1
- data/.ruby-version +1 -0
- data/CHANGELOG.md +7 -0
- data/README.md +10 -0
- data/lib/statsd/instrument/batched_udp_sink.rb +80 -3
- data/lib/statsd/instrument/environment.rb +8 -0
- data/lib/statsd/instrument/version.rb +1 -1
- data/statsd-instrument.gemspec +2 -0
- data/test/dispatcher_stats_test.rb +69 -0
- data/test/udp_sink_test.rb +23 -2
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ce05bd8d34026227e2960ccabca96119580dc2d5737fff81a2bdfd8ea18f826
|
4
|
+
data.tar.gz: d619b08700bb735922673013d7e6314f32d76e8744dcfa07966a35595aa27cf4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 42317b00c680ffc079e89bad712225b8a9656faedf1f8743d9031c26070ae05af0c2ef8a10f1200d94a708023e6cd667d91fbf64a972fecb7d97d327edc31f22
|
7
|
+
data.tar.gz: 540c9a8bccc54633f40e9b2830748e764d6bf5034791bcdacb611be6a7fc615e66a9fa138e2ce58f4a6461f46bb40114fb7507f3f4357d86fccc5d9d6614d244
|
data/.github/workflows/lint.yml
CHANGED
data/.github/workflows/tests.yml
CHANGED
@@ -9,7 +9,7 @@ jobs:
|
|
9
9
|
strategy:
|
10
10
|
fail-fast: false
|
11
11
|
matrix:
|
12
|
-
ruby: ['2.6', '2.7', '3.0', '3.1', '3.2', 'ruby-head', 'jruby-9.3.7.0', 'truffleruby-22.2.0']
|
12
|
+
ruby: ['2.6', '2.7', '3.0', '3.1', '3.2', '3.3', 'ruby-head', 'jruby-9.3.7.0', 'truffleruby-22.2.0']
|
13
13
|
# Windows on macOS builds started failing, so they are disabled for now
|
14
14
|
# platform: [windows-2019, macOS-10.14, ubuntu-18.04]
|
15
15
|
# exclude:
|
data/.rubocop.yml
CHANGED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
3.3.0
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,13 @@ section below.
|
|
6
6
|
|
7
7
|
## Unreleased changes
|
8
8
|
|
9
|
+
## Version 3.8.0
|
10
|
+
|
11
|
+
- UDP batching will now track statistics about its own batching performance, and
|
12
|
+
emit those statistics to the default sink when `STATSD_BATCH_STATISTICS_INTERVAL`
|
13
|
+
is set to any non-zero value. The default value is zero; additional information
|
14
|
+
on statistics tracked is available in the README.
|
15
|
+
|
9
16
|
## Version 3.7.0
|
10
17
|
|
11
18
|
- Add public `.flush` method to sink classes.
|
data/README.md
CHANGED
@@ -54,6 +54,16 @@ The following environment variables are supported:
|
|
54
54
|
If your network is properly configured to handle larger packets you may try
|
55
55
|
to increase this value for better performance, but most network can't handle
|
56
56
|
larger packets.
|
57
|
+
- `STATSD_BATCH_STATISTICS_INTERVAL`: (default: "0") If non-zero, the `BatchedUDPSink`
|
58
|
+
will track and emit statistics on this interval to the default sink for your environment.
|
59
|
+
The current tracked statistics are:
|
60
|
+
|
61
|
+
- `statsd_instrument.batched_udp_sink.batched_sends`: The number of batches sent, of any size.
|
62
|
+
- `statsd_instrument.batched_udp_sink.synchronous_sends`: The number of times the batched udp sender needed to send a statsd line synchronously, due to the buffer being full.
|
63
|
+
- `statsd_instrument.batched_udp_sink.avg_buffer_length`: The average buffer length, measured at the beginning of each batch.
|
64
|
+
- `statsd_instrument.batched_udp_sink.avg_batched_packet_size`: The average per-batch byte size of the packet sent to the underlying UDPSink.
|
65
|
+
- `statsd_instrument.batched_udp_sink.avg_batch_length`: The average number of statsd lines per batch.
|
66
|
+
|
57
67
|
|
58
68
|
## StatsD keys
|
59
69
|
|
@@ -9,6 +9,7 @@ module StatsD
|
|
9
9
|
DEFAULT_BUFFER_CAPACITY = 5_000
|
10
10
|
# https://docs.datadoghq.com/developers/dogstatsd/high_throughput/?code-lang=ruby#ensure-proper-packet-sizes
|
11
11
|
DEFAULT_MAX_PACKET_SIZE = 1472
|
12
|
+
DEFAULT_STATISTICS_INTERVAL = 0 # in seconds, and 0 implies disabled-by-default.
|
12
13
|
|
13
14
|
attr_reader :host, :port
|
14
15
|
|
@@ -28,7 +29,8 @@ module StatsD
|
|
28
29
|
port,
|
29
30
|
thread_priority: DEFAULT_THREAD_PRIORITY,
|
30
31
|
buffer_capacity: DEFAULT_BUFFER_CAPACITY,
|
31
|
-
max_packet_size: DEFAULT_MAX_PACKET_SIZE
|
32
|
+
max_packet_size: DEFAULT_MAX_PACKET_SIZE,
|
33
|
+
statistics_interval: DEFAULT_STATISTICS_INTERVAL
|
32
34
|
)
|
33
35
|
@host = host
|
34
36
|
@port = port
|
@@ -38,6 +40,7 @@ module StatsD
|
|
38
40
|
buffer_capacity,
|
39
41
|
thread_priority,
|
40
42
|
max_packet_size,
|
43
|
+
statistics_interval,
|
41
44
|
)
|
42
45
|
ObjectSpace.define_finalizer(self, self.class.finalize(@dispatcher))
|
43
46
|
end
|
@@ -77,8 +80,70 @@ module StatsD
|
|
77
80
|
end
|
78
81
|
end
|
79
82
|
|
83
|
+
class DispatcherStats
|
84
|
+
def initialize(interval)
|
85
|
+
# The number of times the batched udp sender needed to
|
86
|
+
# send a statsd line synchronously, due to the buffer
|
87
|
+
# being full.
|
88
|
+
@synchronous_sends = 0
|
89
|
+
# The number of times we send a batch of statsd lines,
|
90
|
+
# of any size.
|
91
|
+
@batched_sends = 0
|
92
|
+
# The average buffer length, measured at the beginning of
|
93
|
+
# each batch.
|
94
|
+
@avg_buffer_length = 0
|
95
|
+
# The average per-batch byte size of the packet sent to
|
96
|
+
# the underlying UDPSink.
|
97
|
+
@avg_batched_packet_size = 0
|
98
|
+
# The average number of statsd lines per batch.
|
99
|
+
@avg_batch_length = 0
|
100
|
+
|
101
|
+
@mutex = Mutex.new
|
102
|
+
|
103
|
+
@interval = interval
|
104
|
+
@since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
105
|
+
end
|
106
|
+
|
107
|
+
def maybe_flush!(force: false)
|
108
|
+
return if !force && Process.clock_gettime(Process::CLOCK_MONOTONIC) - @since < @interval
|
109
|
+
|
110
|
+
synchronous_sends = 0
|
111
|
+
batched_sends = 0
|
112
|
+
avg_buffer_length = 0
|
113
|
+
avg_batched_packet_size = 0
|
114
|
+
avg_batch_length = 0
|
115
|
+
@mutex.synchronize do
|
116
|
+
synchronous_sends, @synchronous_sends = @synchronous_sends, synchronous_sends
|
117
|
+
batched_sends, @batched_sends = @batched_sends, batched_sends
|
118
|
+
avg_buffer_length, @avg_buffer_length = @avg_buffer_length, avg_buffer_length
|
119
|
+
avg_batched_packet_size, @avg_batched_packet_size = @avg_batched_packet_size, avg_batched_packet_size
|
120
|
+
avg_batch_length, @avg_batch_length = @avg_batch_length, avg_batch_length
|
121
|
+
@since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
122
|
+
end
|
123
|
+
|
124
|
+
StatsD.increment("statsd_instrument.batched_udp_sink.synchronous_sends", synchronous_sends)
|
125
|
+
StatsD.increment("statsd_instrument.batched_udp_sink.batched_sends", batched_sends)
|
126
|
+
StatsD.gauge("statsd_instrument.batched_udp_sink.avg_buffer_length", avg_buffer_length)
|
127
|
+
StatsD.gauge("statsd_instrument.batched_udp_sink.avg_batched_packet_size", avg_batched_packet_size)
|
128
|
+
StatsD.gauge("statsd_instrument.batched_udp_sink.avg_batch_length", avg_batch_length)
|
129
|
+
end
|
130
|
+
|
131
|
+
def increment_synchronous_sends
|
132
|
+
@mutex.synchronize { @synchronous_sends += 1 }
|
133
|
+
end
|
134
|
+
|
135
|
+
def increment_batched_sends(buffer_len, packet_size, batch_len)
|
136
|
+
@mutex.synchronize do
|
137
|
+
@batched_sends += 1
|
138
|
+
@avg_buffer_length += (buffer_len - @avg_buffer_length) / @batched_sends
|
139
|
+
@avg_batched_packet_size += (packet_size - @avg_batched_packet_size) / @batched_sends
|
140
|
+
@avg_batch_length += (batch_len - @avg_batch_length) / @batched_sends
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
80
145
|
class Dispatcher
|
81
|
-
def initialize(host, port, buffer_capacity, thread_priority, max_packet_size)
|
146
|
+
def initialize(host, port, buffer_capacity, thread_priority, max_packet_size, statistics_interval)
|
82
147
|
@udp_sink = UDPSink.new(host, port)
|
83
148
|
@interrupted = false
|
84
149
|
@thread_priority = thread_priority
|
@@ -87,13 +152,18 @@ module StatsD
|
|
87
152
|
@buffer = Buffer.new(buffer_capacity)
|
88
153
|
@dispatcher_thread = Thread.new { dispatch }
|
89
154
|
@pid = Process.pid
|
155
|
+
if statistics_interval > 0
|
156
|
+
@statistics = DispatcherStats.new(statistics_interval)
|
157
|
+
end
|
90
158
|
end
|
91
159
|
|
92
160
|
def <<(datagram)
|
93
161
|
if !thread_healthcheck || !@buffer.push_nonblock(datagram)
|
94
|
-
# The buffer is full or the thread can't be
|
162
|
+
# The buffer is full or the thread can't be respawned,
|
95
163
|
# we'll send the datagram synchronously
|
96
164
|
@udp_sink << datagram
|
165
|
+
|
166
|
+
@statistics&.increment_synchronous_sends
|
97
167
|
end
|
98
168
|
|
99
169
|
self
|
@@ -119,6 +189,8 @@ module StatsD
|
|
119
189
|
next_datagram ||= @buffer.pop_nonblock
|
120
190
|
break if next_datagram.nil? # no datagram in buffer
|
121
191
|
end
|
192
|
+
buffer_len = @buffer.length + 1
|
193
|
+
batch_len = 1
|
122
194
|
|
123
195
|
packet << next_datagram
|
124
196
|
next_datagram = nil
|
@@ -126,14 +198,19 @@ module StatsD
|
|
126
198
|
while (next_datagram = @buffer.pop_nonblock)
|
127
199
|
if @max_packet_size - packet.bytesize - 1 > next_datagram.bytesize
|
128
200
|
packet << NEWLINE << next_datagram
|
201
|
+
batch_len += 1
|
129
202
|
else
|
130
203
|
break
|
131
204
|
end
|
132
205
|
end
|
133
206
|
end
|
134
207
|
|
208
|
+
packet_size = packet.bytesize
|
135
209
|
@udp_sink << packet
|
136
210
|
packet.clear
|
211
|
+
|
212
|
+
@statistics&.increment_batched_sends(buffer_len, packet_size, batch_len)
|
213
|
+
@statistics&.maybe_flush!
|
137
214
|
end
|
138
215
|
end
|
139
216
|
|
@@ -98,6 +98,13 @@ module StatsD
|
|
98
98
|
Float(env.fetch("STATSD_MAX_PACKET_SIZE", StatsD::Instrument::BatchedUDPSink::DEFAULT_MAX_PACKET_SIZE))
|
99
99
|
end
|
100
100
|
|
101
|
+
def statsd_batch_statistics_interval
|
102
|
+
Integer(env.fetch(
|
103
|
+
"STATSD_BATCH_STATISTICS_INTERVAL",
|
104
|
+
StatsD::Instrument::BatchedUDPSink::DEFAULT_STATISTICS_INTERVAL,
|
105
|
+
))
|
106
|
+
end
|
107
|
+
|
101
108
|
def client
|
102
109
|
StatsD::Instrument::Client.from_env(self)
|
103
110
|
end
|
@@ -110,6 +117,7 @@ module StatsD
|
|
110
117
|
statsd_addr,
|
111
118
|
buffer_capacity: statsd_buffer_capacity,
|
112
119
|
max_packet_size: statsd_max_packet_size,
|
120
|
+
statistics_interval: statsd_batch_statistics_interval,
|
113
121
|
)
|
114
122
|
else
|
115
123
|
StatsD::Instrument::UDPSink.for_addr(statsd_addr)
|
data/statsd-instrument.gemspec
CHANGED
@@ -0,0 +1,69 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "test_helper"
|
4
|
+
|
5
|
+
class DispatcherStatsTest < Minitest::Test
|
6
|
+
include StatsD::Instrument::Assertions
|
7
|
+
|
8
|
+
def test_maybe_flush
|
9
|
+
stats = StatsD::Instrument::BatchedUDPSink::DispatcherStats.new(0)
|
10
|
+
|
11
|
+
stats.increment_synchronous_sends
|
12
|
+
stats.increment_batched_sends(1, 1, 1)
|
13
|
+
|
14
|
+
expectations = [
|
15
|
+
StatsD::Instrument::Expectation.increment("statsd_instrument.batched_udp_sink.synchronous_sends", 1),
|
16
|
+
StatsD::Instrument::Expectation.increment("statsd_instrument.batched_udp_sink.batched_sends", 1),
|
17
|
+
StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_buffer_length", 1),
|
18
|
+
StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_batched_packet_size", 1),
|
19
|
+
StatsD::Instrument::Expectation.gauge("statsd_instrument.batched_udp_sink.avg_batch_length", 1),
|
20
|
+
]
|
21
|
+
assert_statsd_expectations(expectations) { stats.maybe_flush! }
|
22
|
+
assert_equal(0, stats.instance_variable_get(:@synchronous_sends))
|
23
|
+
assert_equal(0, stats.instance_variable_get(:@batched_sends))
|
24
|
+
assert_equal(0, stats.instance_variable_get(:@avg_buffer_length))
|
25
|
+
assert_equal(0, stats.instance_variable_get(:@avg_batched_packet_size))
|
26
|
+
assert_equal(0, stats.instance_variable_get(:@avg_batch_length))
|
27
|
+
|
28
|
+
stats = StatsD::Instrument::BatchedUDPSink::DispatcherStats.new(1)
|
29
|
+
stats.increment_batched_sends(1, 1, 1)
|
30
|
+
assert_no_statsd_calls { stats.maybe_flush! }
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_calculations_are_correct
|
34
|
+
stats = StatsD::Instrument::BatchedUDPSink::DispatcherStats.new(0)
|
35
|
+
|
36
|
+
5.times { stats.increment_synchronous_sends }
|
37
|
+
assert_equal(5, stats.instance_variable_get(:@synchronous_sends))
|
38
|
+
|
39
|
+
batches = [
|
40
|
+
{ buffer_len: 100, packet_size: 1472, batch_len: 10 },
|
41
|
+
{ buffer_len: 90, packet_size: 1300, batch_len: 20 },
|
42
|
+
{ buffer_len: 110, packet_size: 1470, batch_len: 8 },
|
43
|
+
{ buffer_len: 500, packet_size: 1000, batch_len: 1 },
|
44
|
+
{ buffer_len: 100, packet_size: 30, batch_len: 99 },
|
45
|
+
]
|
46
|
+
batches.each do |batch|
|
47
|
+
stats.increment_batched_sends(batch[:buffer_len], batch[:packet_size], batch[:batch_len])
|
48
|
+
end
|
49
|
+
assert_equal(batches.length, stats.instance_variable_get(:@batched_sends))
|
50
|
+
assert_equal(
|
51
|
+
batches.map { |b|
|
52
|
+
b[:buffer_len]
|
53
|
+
}.sum / batches.length,
|
54
|
+
stats.instance_variable_get(:@avg_buffer_length),
|
55
|
+
)
|
56
|
+
assert_equal(
|
57
|
+
batches.map { |b|
|
58
|
+
b[:packet_size]
|
59
|
+
}.sum / batches.length,
|
60
|
+
stats.instance_variable_get(:@avg_batched_packet_size),
|
61
|
+
)
|
62
|
+
assert_equal(
|
63
|
+
batches.map { |b|
|
64
|
+
b[:batch_len]
|
65
|
+
}.sum / batches.length,
|
66
|
+
stats.instance_variable_get(:@avg_batch_length),
|
67
|
+
)
|
68
|
+
end
|
69
|
+
end
|
data/test/udp_sink_test.rb
CHANGED
@@ -202,10 +202,31 @@ class BatchedUDPSinkTest < Minitest::Test
|
|
202
202
|
assert(buffer.empty?)
|
203
203
|
end
|
204
204
|
|
205
|
+
def test_statistics
|
206
|
+
datagrams = StatsD.singleton_client.capture do
|
207
|
+
buffer_size = 2
|
208
|
+
sink = build_sink(@host, @port, buffer_capacity: buffer_size, statistics_interval: 0.1)
|
209
|
+
2.times { |i| sink << "foo:#{i}|c" }
|
210
|
+
sink.flush(blocking: false)
|
211
|
+
sink.instance_variable_get(:@dispatcher).instance_variable_get(:@statistics).maybe_flush!(force: true)
|
212
|
+
end
|
213
|
+
|
214
|
+
assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_batch_length") })
|
215
|
+
assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_batched_packet_size") })
|
216
|
+
assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.avg_buffer_length") })
|
217
|
+
assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.batched_sends") })
|
218
|
+
assert(datagrams.any? { |d| d.name.start_with?("statsd_instrument.batched_udp_sink.synchronous_sends") })
|
219
|
+
end
|
220
|
+
|
205
221
|
private
|
206
222
|
|
207
|
-
def build_sink(host = @host, port = @port, buffer_capacity: 50)
|
208
|
-
sink = @sink_class.new(
|
223
|
+
def build_sink(host = @host, port = @port, buffer_capacity: 50, statistics_interval: 0)
|
224
|
+
sink = @sink_class.new(
|
225
|
+
host,
|
226
|
+
port,
|
227
|
+
buffer_capacity: buffer_capacity,
|
228
|
+
statistics_interval: statistics_interval,
|
229
|
+
)
|
209
230
|
@sinks << sink
|
210
231
|
sink
|
211
232
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: statsd-instrument
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jesse Storimer
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2024-
|
13
|
+
date: 2024-06-19 00:00:00.000000000 Z
|
14
14
|
dependencies: []
|
15
15
|
description: A StatsD client for Ruby apps. Provides metaprogramming methods to inject
|
16
16
|
StatsD instrumentation into your code.
|
@@ -27,6 +27,7 @@ files:
|
|
27
27
|
- ".github/workflows/tests.yml"
|
28
28
|
- ".gitignore"
|
29
29
|
- ".rubocop.yml"
|
30
|
+
- ".ruby-version"
|
30
31
|
- ".yardopts"
|
31
32
|
- CHANGELOG.md
|
32
33
|
- CONTRIBUTING.md
|
@@ -81,6 +82,7 @@ files:
|
|
81
82
|
- test/client_test.rb
|
82
83
|
- test/datagram_builder_test.rb
|
83
84
|
- test/datagram_test.rb
|
85
|
+
- test/dispatcher_stats_test.rb
|
84
86
|
- test/dogstatsd_datagram_builder_test.rb
|
85
87
|
- test/environment_test.rb
|
86
88
|
- test/helpers/rubocop_helper.rb
|
@@ -115,14 +117,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
115
117
|
requirements:
|
116
118
|
- - ">="
|
117
119
|
- !ruby/object:Gem::Version
|
118
|
-
version:
|
120
|
+
version: 2.6.0
|
119
121
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
122
|
requirements:
|
121
123
|
- - ">="
|
122
124
|
- !ruby/object:Gem::Version
|
123
125
|
version: '0'
|
124
126
|
requirements: []
|
125
|
-
rubygems_version: 3.5.
|
127
|
+
rubygems_version: 3.5.11
|
126
128
|
signing_key:
|
127
129
|
specification_version: 4
|
128
130
|
summary: A StatsD client for Ruby apps
|
@@ -136,6 +138,7 @@ test_files:
|
|
136
138
|
- test/client_test.rb
|
137
139
|
- test/datagram_builder_test.rb
|
138
140
|
- test/datagram_test.rb
|
141
|
+
- test/dispatcher_stats_test.rb
|
139
142
|
- test/dogstatsd_datagram_builder_test.rb
|
140
143
|
- test/environment_test.rb
|
141
144
|
- test/helpers/rubocop_helper.rb
|