petef-statsd 0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/statsd +109 -0
- data/lib/statsd.rb +206 -0
- metadata +93 -0
data/bin/statsd
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "rubygems"
|
4
|
+
require "ostruct"
|
5
|
+
require "optparse"
|
6
|
+
require "statsd"
|
7
|
+
|
8
|
+
FLUSH_INTERVAL = 10
|
9
|
+
|
10
|
+
progname = File.basename($0)
|
11
|
+
options = OpenStruct.new
|
12
|
+
options.output_url = nil
|
13
|
+
options.daemonize = false
|
14
|
+
options.port = 8125
|
15
|
+
options.bind = "0.0.0.0"
|
16
|
+
options.pct_threshold = 90
|
17
|
+
options.flush_interval = 10
|
18
|
+
options.key_suffix = nil
|
19
|
+
|
20
|
+
opts = OptionParser.new do |opts|
|
21
|
+
opts.banner = "#{progname} [-d] [-p pct] [-i sec] [-b [host:]port] [-s suffix] -o dest"
|
22
|
+
|
23
|
+
opts.separator "Required flags:"
|
24
|
+
opts.on("-o", "--output=url", String,
|
25
|
+
"Destination for aggregate data") do |x|
|
26
|
+
options.output_url = x
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.separator " output dest should be a URL, example:"
|
30
|
+
opts.separator " * stdout:///"
|
31
|
+
opts.separator " * tcp://graphite.host.com:2003"
|
32
|
+
opts.separator " * amqp://user@vhost:pass@amqp.host.com/type/name"
|
33
|
+
opts.separator " (where type is fanout, queue, or topic)"
|
34
|
+
|
35
|
+
opts.separator ""
|
36
|
+
opts.separator "Optional flags:"
|
37
|
+
|
38
|
+
opts.on("-d", "--[no-]daemonize",
|
39
|
+
"Detach from tty and fork (def false)") do |x|
|
40
|
+
raise NotImplementedError, "--daemonize not implemented yet"
|
41
|
+
#options.daemonize = x
|
42
|
+
end
|
43
|
+
|
44
|
+
opts.on("-b", "--bind=host:port", String,
|
45
|
+
"host:port to bind to for UDP listener (default 0.0.0.0:8125)") do |x|
|
46
|
+
host, port = x.split(":", 2)
|
47
|
+
if ! port # just given a port
|
48
|
+
port = host
|
49
|
+
host = "0.0.0.0"
|
50
|
+
end
|
51
|
+
options.bind = host
|
52
|
+
options.port = port.to_i
|
53
|
+
end
|
54
|
+
|
55
|
+
opts.on("-p", "--percent=threshold", Integer,
|
56
|
+
"% threshold for calculating mean (def 90)") do |x|
|
57
|
+
options.pct_threshold = x
|
58
|
+
end
|
59
|
+
|
60
|
+
opts.on("-i", "--interval=seconds", Integer,
|
61
|
+
"Flush interval in seconds (def 10)") do |x|
|
62
|
+
options.flush_interval = x
|
63
|
+
end
|
64
|
+
|
65
|
+
opts.on("-s", "--suffix=string", String,
|
66
|
+
"suffix to append to all key names") do |x|
|
67
|
+
options.key_suffix = x
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
begin
|
72
|
+
opts.parse!
|
73
|
+
raise "must specify output (-o or --output)" unless options.output_url
|
74
|
+
if options.pct_threshold <= 0 or options.pct_threshold > 100
|
75
|
+
raise "pct_threshold #{options.pct_threshold} out of range, must be 1-100"
|
76
|
+
end
|
77
|
+
if options.port <= 0 or options.pct_threshold > 2**32
|
78
|
+
raise "bind port #{options.port} out of range"
|
79
|
+
end
|
80
|
+
rescue
|
81
|
+
$stderr.puts "#{progname}: #{$!}"
|
82
|
+
$stderr.puts opts
|
83
|
+
exit(1)
|
84
|
+
end
|
85
|
+
|
86
|
+
EM.run do
|
87
|
+
begin
|
88
|
+
StatsD.pct_threshold = options.pct_threshold
|
89
|
+
StatsD.flush_interval = options.flush_interval
|
90
|
+
StatsD.output_url = options.output_url
|
91
|
+
StatsD.key_suffix = options.key_suffix
|
92
|
+
|
93
|
+
EM.add_periodic_timer(options.flush_interval) do
|
94
|
+
EM.defer do
|
95
|
+
begin
|
96
|
+
StatsD.flush
|
97
|
+
rescue
|
98
|
+
StatsD.logger.warn("trouble flushing: #{$!}")
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
EM.open_datagram_socket(options.bind, options.port, StatsD)
|
104
|
+
rescue
|
105
|
+
$stderr.puts "Exception inside of EM.run: #{$!}"
|
106
|
+
EM.stop_event_loop
|
107
|
+
exit 1
|
108
|
+
end
|
109
|
+
end
|
data/lib/statsd.rb
ADDED
@@ -0,0 +1,206 @@
|
|
1
|
+
require "rubygems"
|
2
|
+
require "eventmachine"
|
3
|
+
require "logger"
|
4
|
+
require "socket"
|
5
|
+
require "timeout"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
# Hack because the latest amqp gem uses String#bytesize, and not everyone
|
9
|
+
# is running ruby 1.8.7.
|
10
|
+
if !String.instance_methods.include?(:bytesize)
|
11
|
+
class String
|
12
|
+
alias :bytesize :length
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
module StatsD
|
17
|
+
@@timers = Hash.new { |h, k| h[k] = Array.new }
|
18
|
+
@@counters = Hash.new { |h, k| h[k] = 0 }
|
19
|
+
@@logger = Logger.new(STDERR)
|
20
|
+
@@logger.progname = File.basename($0)
|
21
|
+
@@flush_interval = 10
|
22
|
+
@@pct_threshold = 90
|
23
|
+
@@output_func = :output_stdout
|
24
|
+
@@key_suffix = nil
|
25
|
+
|
26
|
+
def self.logger; return @@logger; end
|
27
|
+
def self.logger_output=(output)
|
28
|
+
@@logger = Logger.new(output)
|
29
|
+
@@logger.progname = File.basename($0)
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.flush_interval=(val)
|
33
|
+
@@flush_interval = val.to_i
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.pct_threshold=(val)
|
37
|
+
@@pct_threshold = val.to_i
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.key_suffix=(val)
|
41
|
+
@@key_suffix = val
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.output_url=(url)
|
45
|
+
@@output_url = URI.parse(url)
|
46
|
+
scheme_mapper = {"tcp" => [nil, :output_tcp],
|
47
|
+
"amqp" => [:setup_amqp, :output_amqp],
|
48
|
+
"stdout" => [nil, :output_stdout],
|
49
|
+
}
|
50
|
+
if ! scheme_mapper.has_key?(@@output_url.scheme)
|
51
|
+
raise TypeError, "unsupported scheme in #{url}"
|
52
|
+
end
|
53
|
+
|
54
|
+
setup_func, @@output_func = scheme_mapper[@@output_url.scheme]
|
55
|
+
self.send(setup_func) if setup_func
|
56
|
+
end
|
57
|
+
|
58
|
+
# TODO: option for persistent tcp connection
|
59
|
+
#def setup_tcp
|
60
|
+
#end
|
61
|
+
|
62
|
+
def self.output_tcp(packet)
|
63
|
+
server = TCPSocket.new(@@output_url.host, @@output_url.port)
|
64
|
+
server.puts packet
|
65
|
+
server.close
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.setup_amqp
|
69
|
+
begin
|
70
|
+
require "amqp"
|
71
|
+
require "mq"
|
72
|
+
rescue LoadError
|
73
|
+
@@logger.fatal("missing amqp ruby module. try gem install amqp")
|
74
|
+
exit(1)
|
75
|
+
end
|
76
|
+
|
77
|
+
user = @@output_url.user || ""
|
78
|
+
user, vhost = user.split("@", 2)
|
79
|
+
_, mqtype, mqname = @@output_url.path.split("/", 3)
|
80
|
+
amqp_settings = {
|
81
|
+
:host => @@output_url.host,
|
82
|
+
:port => @@output_url.port || 5672,
|
83
|
+
:user => user,
|
84
|
+
:pass => @@output_url.password,
|
85
|
+
:vhost => vhost || "/",
|
86
|
+
}
|
87
|
+
|
88
|
+
@amqp = AMQP.connect(amqp_settings)
|
89
|
+
@mq = MQ.new(@amqp)
|
90
|
+
@target = nil
|
91
|
+
opts = {:durable => true,
|
92
|
+
:auto_delete => false,
|
93
|
+
}
|
94
|
+
|
95
|
+
if @@output_url.query
|
96
|
+
@@output_url.query.split("&").each do |param|
|
97
|
+
k, v = param.split("=", 2)
|
98
|
+
opts[:durable] = false if k == "durable" and v == "false"
|
99
|
+
opts[:auto_delete] = true if k == "autodelete" and v == "true"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
@@logger.info(opts.inspect)
|
104
|
+
|
105
|
+
case mqtype
|
106
|
+
when "fanout"
|
107
|
+
@target = @mq.fanout(mqname, opts)
|
108
|
+
when "queue"
|
109
|
+
@target = @mq.queue(mqname, opts)
|
110
|
+
when "topic"
|
111
|
+
@target = @mq.topic(mqname, opts)
|
112
|
+
else
|
113
|
+
raise TypeError, "unknown mq output type #{mqname}"
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.output_amqp(packet)
|
118
|
+
@target.publish(packet)
|
119
|
+
end
|
120
|
+
|
121
|
+
def self.output_stdout(packet)
|
122
|
+
$stdout.write(packet)
|
123
|
+
end
|
124
|
+
|
125
|
+
def receive_data(packet)
|
126
|
+
bits = packet.split(":")
|
127
|
+
key = bits.shift.gsub(/\s+/, "_") \
|
128
|
+
.gsub(/\//, "-") \
|
129
|
+
.gsub(/[^a-zA-Z_\-0-9\.]/, "")
|
130
|
+
bits << "1" if bits.length == 0
|
131
|
+
bits.each do |bit|
|
132
|
+
fields = bit.split("|")
|
133
|
+
if fields.length != 2
|
134
|
+
$stderr.puts "invalid update: #{bit}"
|
135
|
+
next
|
136
|
+
end
|
137
|
+
|
138
|
+
if fields[1] == "ms" # timer update
|
139
|
+
@@timers[key] << fields[0].to_f
|
140
|
+
elsif fields[1] == "c" # counter update
|
141
|
+
count, sample_rate = fields[0].split("@", 2)
|
142
|
+
sample_rate ||= 1
|
143
|
+
#puts "count is #{count.to_f} (#{count})"
|
144
|
+
#puts "multiplier is is #{1 / sample_rate.to_f}"
|
145
|
+
@@counters[key] += count.to_f * (1 / sample_rate.to_f)
|
146
|
+
else
|
147
|
+
$stderr.puts "invalid field in update: #{bit}"
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def self.carbon_update_str
|
153
|
+
updates = []
|
154
|
+
now = Time.now.to_i
|
155
|
+
|
156
|
+
@@timers.each do |key, values|
|
157
|
+
next if values.length == 0
|
158
|
+
values.sort!
|
159
|
+
min = values[0]
|
160
|
+
max = values[-1]
|
161
|
+
mean = min
|
162
|
+
maxAtThreshold = min
|
163
|
+
if values.length > 1
|
164
|
+
threshold_index = ((100 - @@pct_threshold) / 100.0) * values.length
|
165
|
+
threshold_count = values.length - threshold_index.round
|
166
|
+
valid_values = values.slice(0, threshold_count)
|
167
|
+
maxAtThreshold = valid_values[-1]
|
168
|
+
|
169
|
+
sum = 0
|
170
|
+
valid_values.each { |v| sum += v }
|
171
|
+
mean = sum / valid_values.length
|
172
|
+
end
|
173
|
+
|
174
|
+
suffix = @@key_suffix ? ".#{@@key_suffix}" : ""
|
175
|
+
updates << "stats.timers.#{key}.mean#{suffix} #{mean} #{now}"
|
176
|
+
updates << "stats.timers.#{key}.upper#{suffix} #{max} #{now}"
|
177
|
+
updates << "stats.timers.#{key}.upper_#{@@pct_threshold}#{suffix} " \
|
178
|
+
"#{maxAtThreshold} #{now}"
|
179
|
+
updates << "stats.timers.#{key}.lower#{suffix} #{min} #{now}"
|
180
|
+
updates << "stats.timers.#{key}.count#{suffix} #{values.length} #{now}"
|
181
|
+
end
|
182
|
+
|
183
|
+
@@counters.each do |key, value|
|
184
|
+
suffix = @@key_suffix ? ".#{@@key_suffix}" : ""
|
185
|
+
updates << "stats.#{key}#{suffix} #{value / @@flush_interval} #{now}"
|
186
|
+
end
|
187
|
+
|
188
|
+
@@timers.each { |k, v| @@timers[k] = [] }
|
189
|
+
@@counters.each { |k, v| @@counters[k] = 0 }
|
190
|
+
|
191
|
+
return updates.length == 0 ? nil : updates.join("\n") + "\n"
|
192
|
+
end
|
193
|
+
|
194
|
+
def self.flush
|
195
|
+
s = carbon_update_str
|
196
|
+
return unless s
|
197
|
+
|
198
|
+
begin
|
199
|
+
Timeout::timeout(2) { self.send(@@output_func, s) }
|
200
|
+
rescue Timeout::Error
|
201
|
+
@@logger.warn("timed out sending update to #{@@output_url}")
|
202
|
+
rescue
|
203
|
+
@@logger.warn("error sending update to #{@@output_url}: #{$!}")
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
metadata
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: petef-statsd
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 13
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 3
|
9
|
+
version: "0.3"
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Pete Fritchman
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2011-12-02 00:00:00 Z
|
18
|
+
dependencies:
|
19
|
+
- !ruby/object:Gem::Dependency
|
20
|
+
name: eventmachine
|
21
|
+
prerelease: false
|
22
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
23
|
+
none: false
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
hash: 3
|
28
|
+
segments:
|
29
|
+
- 0
|
30
|
+
version: "0"
|
31
|
+
type: :runtime
|
32
|
+
version_requirements: *id001
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: amqp
|
35
|
+
prerelease: false
|
36
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
37
|
+
none: false
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
hash: 3
|
42
|
+
segments:
|
43
|
+
- 0
|
44
|
+
version: "0"
|
45
|
+
type: :runtime
|
46
|
+
version_requirements: *id002
|
47
|
+
description: collect and aggregate stats, flush to graphite
|
48
|
+
email: petef@databits.net
|
49
|
+
executables:
|
50
|
+
- statsd
|
51
|
+
extensions: []
|
52
|
+
|
53
|
+
extra_rdoc_files: []
|
54
|
+
|
55
|
+
files:
|
56
|
+
- lib/statsd.rb
|
57
|
+
- bin/statsd
|
58
|
+
homepage: https://github.com/fetep/ruby-statsd
|
59
|
+
licenses:
|
60
|
+
- Mozilla Public License (1.1)
|
61
|
+
post_install_message:
|
62
|
+
rdoc_options: []
|
63
|
+
|
64
|
+
require_paths:
|
65
|
+
- lib
|
66
|
+
- lib
|
67
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
68
|
+
none: false
|
69
|
+
requirements:
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
hash: 3
|
73
|
+
segments:
|
74
|
+
- 0
|
75
|
+
version: "0"
|
76
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
77
|
+
none: false
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
hash: 3
|
82
|
+
segments:
|
83
|
+
- 0
|
84
|
+
version: "0"
|
85
|
+
requirements: []
|
86
|
+
|
87
|
+
rubyforge_project:
|
88
|
+
rubygems_version: 1.8.10
|
89
|
+
signing_key:
|
90
|
+
specification_version: 3
|
91
|
+
summary: statsd -- stat collector/aggregator
|
92
|
+
test_files: []
|
93
|
+
|