graphite-api 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +14 -18
- data/bin/graphite-middleware +68 -1
- data/lib/graphite-api.rb +7 -14
- data/lib/graphite-api/buffer.rb +16 -17
- data/lib/graphite-api/cache.rb +32 -2
- data/lib/graphite-api/client.rb +25 -24
- data/lib/graphite-api/connector.rb +77 -17
- data/lib/graphite-api/logger.rb +3 -5
- data/lib/graphite-api/middleware.rb +6 -10
- data/lib/graphite-api/version.rb +1 -1
- metadata +3 -7
- data/lib/graphite-api/cache/memory.rb +0 -36
- data/lib/graphite-api/cli.rb +0 -63
- data/lib/graphite-api/runner.rb +0 -52
- data/lib/graphite-api/utils.rb +0 -70
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e7f26c3b31d445c4e11d556609f0c3b600c60fe7
|
4
|
+
data.tar.gz: 657df67acbef98202c7278e473994b4bd47841f9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4e464886000e85f88d21dd6351d60e3160a3dc9b4e66b4d98f72f7fd3ae1f50556494843c21b3430d1d26aa11ed2c7c9d23dcf36bd8b7177a1dc804e07811d91
|
7
|
+
data.tar.gz: 8cba41ac302987b045af13b8ef025ff08b9c93e49353442e38edbd7b8db66ec70c4f538ec4f6aff4d125e2f777972aef5bc1b7b5a20b2104863e1803573e183a
|
data/README.md
CHANGED
@@ -1,13 +1,9 @@
|
|
1
|
-
#
|
2
|
-
A Ruby API toolkit for [Graphite](http://graphite.wikidot.com/).
|
3
|
-
|
4
|
-
## Description
|
1
|
+
# Description
|
5
2
|
**GraphiteAPI** provides two ways for interacting with **Graphite's Carbon Daemon**, the first is for Ruby applications using the **GraphiteAPI::Client**, the second is through **GraphiteAPI-Middleware** daemon, both methods implements Graphite's [plaintext protocol](http://graphite.readthedocs.org/en/1.0/feeding-carbon.html).
|
6
3
|
|
7
4
|
## Package Content
|
8
5
|
* Includes a **simple** client for ruby.
|
9
6
|
* Ships with a **GraphiteAPI-Middleware**, which is a lightweight, event-driven, aggregator daemon.
|
10
|
-
* Only one gem dependency ( EventMachine ).
|
11
7
|
* Utilities like scheduling and caching.
|
12
8
|
|
13
9
|
## Key Features
|
@@ -51,12 +47,12 @@ Creating a new client instance
|
|
51
47
|
require 'graphite-api'
|
52
48
|
|
53
49
|
GraphiteAPI.new(
|
54
|
-
graphite: "graphite.example.com:2003", # required argument
|
55
|
-
prefix: ["example","prefix"],
|
56
|
-
slice: 60,
|
57
|
-
interval: 60,
|
58
|
-
|
59
|
-
cache: 4 * 60 * 60
|
50
|
+
graphite: "udp://graphite.example.com:2003", # required argument
|
51
|
+
prefix: ["example","prefix"], # add example.prefix to each key
|
52
|
+
slice: 60, # results are aggregated in 60 seconds slices
|
53
|
+
interval: 60, # send to graphite every 60 seconds
|
54
|
+
# default is 0 ( direct send )
|
55
|
+
cache: 4 * 60 * 60 # set the max age in seconds for records reanimation
|
60
56
|
)
|
61
57
|
```
|
62
58
|
|
@@ -64,7 +60,7 @@ Adding simple metrics
|
|
64
60
|
```ruby
|
65
61
|
require 'graphite-api'
|
66
62
|
|
67
|
-
client = GraphiteAPI.new( graphite: 'graphite:2003' )
|
63
|
+
client = GraphiteAPI.new( graphite: 'tcp://graphite:2003' )
|
68
64
|
|
69
65
|
client.metrics "webServer.web01.loadAvg" => 10.7
|
70
66
|
# => webServer.web01.loadAvg 10.7 time.now.to_i
|
@@ -81,7 +77,7 @@ Adding metrics with timestamp
|
|
81
77
|
```ruby
|
82
78
|
require 'graphite-api'
|
83
79
|
|
84
|
-
client = GraphiteAPI.new( graphite: 'graphite:2003' )
|
80
|
+
client = GraphiteAPI.new( graphite: 'udp://graphite:2003' )
|
85
81
|
|
86
82
|
client.metrics({
|
87
83
|
"webServer.web01.loadAvg" => 10.7,
|
@@ -95,7 +91,7 @@ Increment records
|
|
95
91
|
```ruby
|
96
92
|
require 'graphite-api'
|
97
93
|
|
98
|
-
client = GraphiteAPI.new( graphite: 'graphite:2003' )
|
94
|
+
client = GraphiteAPI.new( graphite: 'tcp://graphite:2003' )
|
99
95
|
|
100
96
|
client.increment("jobs_in_queue", "num_errors")
|
101
97
|
# => jobs_in_queue 1 Time.now.to_i
|
@@ -115,7 +111,7 @@ Built-in timers support
|
|
115
111
|
```ruby
|
116
112
|
require 'graphite-api'
|
117
113
|
|
118
|
-
client = GraphiteAPI.new( graphite: 'graphite:2003' )
|
114
|
+
client = GraphiteAPI.new( graphite: 'tcp://graphite:2003' )
|
119
115
|
|
120
116
|
# lets send the metric every 120 seconds
|
121
117
|
client.every(120) do |c|
|
@@ -128,7 +124,7 @@ Built-in extension for time declarations stuff, like 2.minutes, 3.hours etc...
|
|
128
124
|
require 'graphite-api'
|
129
125
|
require 'graphite-api/core_ext/numeric'
|
130
126
|
|
131
|
-
client = GraphiteAPI.new( graphite: 'graphite:2003' )
|
127
|
+
client = GraphiteAPI.new( graphite: 'udp://graphite:2003' )
|
132
128
|
|
133
129
|
client.every 10.seconds do |c|
|
134
130
|
c.metrics("webServer.web01.uptime" => `uptime`.split.first.to_i)
|
@@ -144,7 +140,7 @@ Make your own custom metrics daemons, using `client#join`
|
|
144
140
|
require 'graphite-api'
|
145
141
|
require 'graphite-api/core_ext/numeric'
|
146
142
|
|
147
|
-
client = GraphiteAPI.new( graphite: 'graphite:2003' )
|
143
|
+
client = GraphiteAPI.new( graphite: 'udp://graphite:2003' )
|
148
144
|
|
149
145
|
client.every 26.minutes do |c|
|
150
146
|
c.metrics("webServer.shuki.stats" => 10)
|
@@ -220,7 +216,7 @@ example.middleware.value2 99 1334929231
|
|
220
216
|
|
221
217
|
```ruby
|
222
218
|
require 'graphite-api'
|
223
|
-
client = GraphiteAPI.new(:graphite => 'graphite-middleware-node:2005')
|
219
|
+
client = GraphiteAPI.new(:graphite => 'tcp://graphite-middleware-node:2005')
|
224
220
|
client.example.middleware.value 10.2
|
225
221
|
client.example.middleware.value2 27
|
226
222
|
client.bla.bla.value2 27
|
data/bin/graphite-middleware
CHANGED
@@ -2,5 +2,72 @@
|
|
2
2
|
$:.unshift File.expand_path('../../lib',__FILE__)
|
3
3
|
|
4
4
|
require 'graphite-api'
|
5
|
+
require 'optparse'
|
5
6
|
|
6
|
-
GraphiteAPI::
|
7
|
+
options = GraphiteAPI::Client.default_options.merge interval: 60
|
8
|
+
|
9
|
+
OptionParser.new do |opts|
|
10
|
+
opts.banner = "GraphiteAPI Middleware Server"
|
11
|
+
opts.define_head "Usage: graphite-middleware [options]"
|
12
|
+
opts.define_head ""
|
13
|
+
|
14
|
+
opts.on("-g", "--graphite [udp,tcp]://host:port", "can be specified multiple times") do |graphite|
|
15
|
+
options[:backends].push graphite
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on("-p", "--port PORT","listening port (default #{options[:port]})") do |port|
|
19
|
+
options[:port] = port
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on("-l", "--log-file FILE","log file") do |file|
|
23
|
+
options[:log_file] = File::expand_path(file)
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on("-L", "--log-level LEVEL","log level (default warn)") do |level|
|
27
|
+
options[:log_level] = level
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on("-P", "--pid-file FILE","pid file (default #{options[:pid]})") do |pid_file|
|
31
|
+
options[:pid] = pid_file
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on("-d", "--daemonize","run in background") do
|
35
|
+
options[:daemonize] = true
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on("-i", "--interval INT","report every X seconds (default #{options[:interval]})") do |x_seconds|
|
39
|
+
options[:interval] = x_seconds.to_i if x_seconds.to_i > 0
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.on("-s", "--slice SECONDS","send to graphite in X seconds slices (default #{options[:slice]})") do |slice|
|
43
|
+
options[:slice] = slice.to_i if slice.to_i >= 0
|
44
|
+
end
|
45
|
+
|
46
|
+
opts.on("-r", "--reanimation HOURS","reanimate records that are younger than X hours, please see README") do |exp|
|
47
|
+
(options[:cache] = exp.to_i * 3600) if exp.to_i > 0
|
48
|
+
end
|
49
|
+
|
50
|
+
opts.on("-v", "--version","Show version and exit") do |exp|
|
51
|
+
puts "Version #{GraphiteAPI.version}"
|
52
|
+
exit
|
53
|
+
end
|
54
|
+
|
55
|
+
opts.separator ""
|
56
|
+
opts.separator "More Info @ https://github.com/kontera-technologies/graphite-api"
|
57
|
+
opts.separator ""
|
58
|
+
|
59
|
+
opts.define_tail ""
|
60
|
+
opts.define_tail ""
|
61
|
+
end.parse! ARGV
|
62
|
+
|
63
|
+
abort "You must specify at least one graphite host" if options[:backends].empty?
|
64
|
+
|
65
|
+
GraphiteAPI::Logger.init Hash[[:dev,:level].zip options.values_at(:log_file, :log_level) ]
|
66
|
+
|
67
|
+
begin
|
68
|
+
Process.daemon if options[:daemonize]
|
69
|
+
GraphiteAPI::Middleware.start options
|
70
|
+
rescue Interrupt
|
71
|
+
GraphiteAPI::Logger.info "Shutting down..."
|
72
|
+
Zscheduler.stop
|
73
|
+
end
|
data/lib/graphite-api.rb
CHANGED
@@ -1,20 +1,13 @@
|
|
1
1
|
require 'zscheduler'
|
2
|
+
require 'graphite-api/version'
|
3
|
+
require 'graphite-api/client'
|
4
|
+
require 'graphite-api/cache'
|
5
|
+
require 'graphite-api/connector'
|
6
|
+
require 'graphite-api/middleware'
|
7
|
+
require 'graphite-api/buffer'
|
8
|
+
require 'graphite-api/logger'
|
2
9
|
|
3
10
|
module GraphiteAPI
|
4
|
-
ROOT = File.expand_path File.dirname __FILE__
|
5
|
-
|
6
|
-
require "#{ROOT}/graphite-api/version"
|
7
|
-
|
8
|
-
autoload :Version, "#{ROOT}/graphite-api/version"
|
9
|
-
autoload :Client, "#{ROOT}/graphite-api/client"
|
10
|
-
autoload :Cache, "#{ROOT}/graphite-api/cache"
|
11
|
-
autoload :Connector, "#{ROOT}/graphite-api/connector"
|
12
|
-
autoload :Middleware, "#{ROOT}/graphite-api/middleware"
|
13
|
-
autoload :Runner, "#{ROOT}/graphite-api/runner"
|
14
|
-
autoload :Utils, "#{ROOT}/graphite-api/utils"
|
15
|
-
autoload :CLI, "#{ROOT}/graphite-api/cli"
|
16
|
-
autoload :Buffer, "#{ROOT}/graphite-api/buffer"
|
17
|
-
autoload :Logger, "#{ROOT}/graphite-api/logger"
|
18
11
|
|
19
12
|
def self.version
|
20
13
|
GraphiteAPI::VERSION
|
data/lib/graphite-api/buffer.rb
CHANGED
@@ -21,9 +21,8 @@ require 'set'
|
|
21
21
|
|
22
22
|
module GraphiteAPI
|
23
23
|
class Buffer
|
24
|
-
include Utils
|
25
24
|
|
26
|
-
|
25
|
+
IGNORE = ["\r"]
|
27
26
|
END_OF_STREAM = "\n"
|
28
27
|
VALID_MESSAGE = /^[\w|\.|-]+ \d+(?:\.|\d)* \d+$/
|
29
28
|
|
@@ -34,7 +33,7 @@ module GraphiteAPI
|
|
34
33
|
@cache = Cache::Memory.new options if options[:cache]
|
35
34
|
end
|
36
35
|
|
37
|
-
|
36
|
+
attr_reader :queue, :options, :streamer, :cache
|
38
37
|
|
39
38
|
# this method isn't thread safe
|
40
39
|
# use #push for multiple threads support
|
@@ -44,7 +43,7 @@ module GraphiteAPI
|
|
44
43
|
streamer[client_id] += char
|
45
44
|
|
46
45
|
if closed_stream? streamer[client_id]
|
47
|
-
if
|
46
|
+
if streamer[client_id] =~ VALID_MESSAGE
|
48
47
|
push stream_message_to_obj streamer[client_id]
|
49
48
|
end
|
50
49
|
streamer.delete client_id
|
@@ -55,7 +54,7 @@ module GraphiteAPI
|
|
55
54
|
# Add records to buffer
|
56
55
|
# push({:metric => {'a' => 10},:time => Time.now})
|
57
56
|
def push obj
|
58
|
-
debug [:buffer,:add, obj]
|
57
|
+
Logger.debug [:buffer,:add, obj]
|
59
58
|
queue.push obj
|
60
59
|
nil
|
61
60
|
end
|
@@ -63,14 +62,13 @@ module GraphiteAPI
|
|
63
62
|
alias_method :<<, :push
|
64
63
|
|
65
64
|
def pull format = nil
|
66
|
-
data =
|
65
|
+
data = Hash.new {|h,k| h[k] = Hash.new {|h2,k2| h2[k2] = 0}}
|
67
66
|
|
68
67
|
counter = 0
|
69
68
|
while new_records?
|
70
69
|
break if ( counter += 1 ) > 1_000_000 # TODO: fix this
|
71
70
|
hash = queue.pop
|
72
|
-
|
73
|
-
hash[:metric].each { |k,v| data[time][k] += v.to_f }
|
71
|
+
hash[:metric].each {|k,v| data[normalize_time(hash[:time],options[:slice])][k] += v.to_f}
|
74
72
|
end
|
75
73
|
|
76
74
|
data.map do |time, hash|
|
@@ -82,16 +80,21 @@ module GraphiteAPI
|
|
82
80
|
end.flatten(1)
|
83
81
|
end
|
84
82
|
|
85
|
-
def new_records?
|
86
|
-
!queue.empty?
|
87
|
-
end
|
88
|
-
|
89
83
|
def inspect
|
90
84
|
"#<GraphiteAPI::Buffer:%s @quque#size=%s @streamer=%s>" %
|
91
85
|
[ object_id, queue.size, streamer]
|
92
86
|
end
|
93
87
|
|
88
|
+
def new_records?
|
89
|
+
!queue.empty?
|
90
|
+
end
|
91
|
+
|
94
92
|
private
|
93
|
+
|
94
|
+
def normalize_time time, slice
|
95
|
+
slice = 60 if slice.nil?
|
96
|
+
((time || Time.now).to_i / slice * slice).to_i
|
97
|
+
end
|
95
98
|
|
96
99
|
def stream_message_to_obj message
|
97
100
|
parts = message.split
|
@@ -99,16 +102,12 @@ module GraphiteAPI
|
|
99
102
|
end
|
100
103
|
|
101
104
|
def invalid_char? char
|
102
|
-
|
105
|
+
IGNORE.include? char
|
103
106
|
end
|
104
107
|
|
105
108
|
def closed_stream? string
|
106
109
|
string[-1,1] == END_OF_STREAM
|
107
110
|
end
|
108
|
-
|
109
|
-
def valid_stream_message? message
|
110
|
-
message =~ VALID_MESSAGE
|
111
|
-
end
|
112
111
|
|
113
112
|
def prefix
|
114
113
|
@prefix ||= if options[:prefix] and !options[:prefix].empty?
|
data/lib/graphite-api/cache.rb
CHANGED
@@ -1,5 +1,35 @@
|
|
1
1
|
module GraphiteAPI
|
2
2
|
module Cache
|
3
|
-
|
3
|
+
class Memory
|
4
|
+
|
5
|
+
def initialize options
|
6
|
+
Zscheduler.every(120) { clean(options[:cache]) }
|
7
|
+
end
|
8
|
+
|
9
|
+
def get time, key
|
10
|
+
cache[time.to_i][key]
|
11
|
+
end
|
12
|
+
|
13
|
+
def set time, key, value
|
14
|
+
cache[time.to_i][key] = value.to_f
|
15
|
+
end
|
16
|
+
|
17
|
+
def incr time, key, value
|
18
|
+
set(time, key, value.to_f + get(time, key))
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def cache
|
24
|
+
@cache ||= Hash.new {|h,k| h[k] = Hash.new {|h2,k2| h2[k2] = 0}}
|
25
|
+
end
|
26
|
+
|
27
|
+
def clean max_age
|
28
|
+
Logger.debug [:MemoryCache, :before_clean, cache]
|
29
|
+
cache.delete_if {|t,k| Time.now.to_i - t > max_age }
|
30
|
+
Logger.debug [:MemoryCache, :after_clean, cache]
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
4
34
|
end
|
5
|
-
end
|
35
|
+
end
|
data/lib/graphite-api/client.rb
CHANGED
@@ -1,10 +1,13 @@
|
|
1
|
-
require
|
2
|
-
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
3
|
module GraphiteAPI
|
4
4
|
class Client
|
5
|
-
|
5
|
+
extend Forwardable
|
6
6
|
|
7
|
-
|
7
|
+
def_delegator Zscheduler, :loop, :join
|
8
|
+
def_delegator Zscheduler, :stop
|
9
|
+
|
10
|
+
attr_reader :options, :buffer, :connectors
|
8
11
|
|
9
12
|
def initialize opt
|
10
13
|
@options = build_options validate opt.clone
|
@@ -14,9 +17,6 @@ module GraphiteAPI
|
|
14
17
|
Zscheduler.every(options[:interval]) { send_metrics } unless options[:direct]
|
15
18
|
end
|
16
19
|
|
17
|
-
def_delegator Zscheduler, :loop, :join
|
18
|
-
def_delegator Zscheduler, :stop
|
19
|
-
|
20
20
|
def every interval, &block
|
21
21
|
Zscheduler.every( interval ) { block.arity == 1 ? block.call(self) : block.call }
|
22
22
|
end
|
@@ -27,24 +27,12 @@ module GraphiteAPI
|
|
27
27
|
send_metrics if options[:direct]
|
28
28
|
end
|
29
29
|
|
30
|
-
alias_method :add_metrics, :metrics
|
31
|
-
|
32
|
-
# increment keys
|
33
|
-
#
|
34
|
-
# increment("key1","key2")
|
35
|
-
# => metrics("key1" => 1, "key2" => 1)
|
36
|
-
#
|
37
|
-
# increment("key1","key2", {:by => 999})
|
38
|
-
# => metrics("key1" => 999, "key2" => 999)
|
39
|
-
#
|
40
|
-
# increment("key1","key2", {:time => Time.at(123456)})
|
41
|
-
# => metrics({"key1" => 1, "key2" => 1},Time.at(123456))
|
42
30
|
def increment(*keys)
|
43
31
|
opt = {}
|
44
32
|
opt.merge! keys.pop if keys.last.is_a? Hash
|
45
33
|
by = opt.fetch(:by,1)
|
46
34
|
time = opt.fetch(:time,Time.now)
|
47
|
-
metric = keys.inject({}) {|h,k| h.
|
35
|
+
metric = keys.inject({}) {|h,k| h.merge k => by }
|
48
36
|
metrics(metric, time)
|
49
37
|
end
|
50
38
|
|
@@ -52,6 +40,21 @@ module GraphiteAPI
|
|
52
40
|
sleep while buffer.new_records?
|
53
41
|
end
|
54
42
|
|
43
|
+
def self.default_options
|
44
|
+
{
|
45
|
+
:backends => [],
|
46
|
+
:cleaner_interval => 43200,
|
47
|
+
:port => 2003,
|
48
|
+
:log_level => :info,
|
49
|
+
:cache => nil,
|
50
|
+
:host => "localhost",
|
51
|
+
:prefix => [],
|
52
|
+
:interval => 0,
|
53
|
+
:slice => 60,
|
54
|
+
:pid => "/tmp/graphite-middleware.pid"
|
55
|
+
}
|
56
|
+
end
|
57
|
+
|
55
58
|
protected
|
56
59
|
|
57
60
|
def validate options
|
@@ -61,8 +64,8 @@ module GraphiteAPI
|
|
61
64
|
end
|
62
65
|
|
63
66
|
def build_options opt
|
64
|
-
default_options.tap do |options_hash|
|
65
|
-
options_hash[:backends].push
|
67
|
+
self.class.default_options.tap do |options_hash|
|
68
|
+
options_hash[:backends].push opt.delete :graphite
|
66
69
|
options_hash.merge! opt
|
67
70
|
options_hash[:direct] = options_hash[:interval] == 0
|
68
71
|
options_hash[:slice] = 1 if options_hash[:direct]
|
@@ -71,8 +74,6 @@ module GraphiteAPI
|
|
71
74
|
|
72
75
|
def send_metrics
|
73
76
|
connectors.publish buffer.pull :string if buffer.new_records?
|
74
|
-
rescue Exception => e
|
75
|
-
Zscheduler.init_reactor? ? raise : Logger.error("Publish Error: #{e}\n#{e.backtrace.join("\n")}")
|
76
77
|
end
|
77
78
|
|
78
79
|
end
|
@@ -9,48 +9,108 @@
|
|
9
9
|
# => my.metric 1092 1232123231\n
|
10
10
|
# -----------------------------------------------------
|
11
11
|
require 'socket'
|
12
|
+
require 'uri'
|
12
13
|
|
13
14
|
module GraphiteAPI
|
14
15
|
class Connector
|
16
|
+
|
17
|
+
class UDPSocket
|
18
|
+
def initialize uri
|
19
|
+
@uri = uri
|
20
|
+
@socket = ::UDPSocket.new
|
21
|
+
end
|
22
|
+
|
23
|
+
def puts message
|
24
|
+
@socket.send message, 0, @uri.host, @uri.port
|
25
|
+
end
|
26
|
+
|
27
|
+
def closed?
|
28
|
+
@socket.closed? rescue true
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class TCPSocket
|
33
|
+
def initialize uri
|
34
|
+
@uri = uri
|
35
|
+
end
|
36
|
+
|
37
|
+
def puts message
|
38
|
+
socket.puts message
|
39
|
+
end
|
40
|
+
|
41
|
+
def closed?
|
42
|
+
socket.closed? rescue true
|
43
|
+
end
|
44
|
+
|
45
|
+
def socket
|
46
|
+
@socket ||= begin
|
47
|
+
host, port = @uri.host, @uri.port
|
48
|
+
timeout = Hash[URI.decode_www_form(@uri.query.to_s)].fetch("timeout", 1)
|
49
|
+
addr = Socket.getaddrinfo host, nil, :INET
|
50
|
+
sockaddr = Socket.pack_sockaddr_in port, addr[0][3]
|
51
|
+
|
52
|
+
sock = Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0)
|
53
|
+
sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
|
54
|
+
begin
|
55
|
+
sock.connect_nonblock sockaddr
|
56
|
+
rescue IO::WaitWritable
|
57
|
+
if IO.select nil, [sock], nil, timeout
|
58
|
+
begin
|
59
|
+
sock.connect_nonblock sockaddr
|
60
|
+
rescue Errno::EISCONN # success
|
61
|
+
rescue
|
62
|
+
sock.close
|
63
|
+
raise
|
64
|
+
end
|
65
|
+
else
|
66
|
+
sock.close
|
67
|
+
raise "Connection timeout"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
sock
|
71
|
+
end #begin
|
72
|
+
|
73
|
+
end # Socket
|
74
|
+
end # TCPSocket
|
75
|
+
|
15
76
|
class Group
|
16
77
|
def initialize options
|
17
|
-
@connectors = options[:backends].map
|
78
|
+
@connectors = options[:backends].map(&Connector.method(:new))
|
18
79
|
end
|
19
80
|
|
20
81
|
def publish messages
|
21
|
-
Logger.debug [:connector_group, :publish, messages
|
82
|
+
Logger.debug [:connector_group, :publish, messages, @connectors]
|
22
83
|
Array(messages).each { |msg| @connectors.map {|c| c.puts msg} }
|
23
84
|
end
|
24
85
|
end
|
25
|
-
|
26
|
-
|
27
|
-
|
86
|
+
|
87
|
+
##############################################################################
|
88
|
+
|
89
|
+
def initialize uri
|
90
|
+
@uri = URI.parse uri
|
91
|
+
@uri = @uri.host ? @uri : URI.parse("udp://#{uri}")
|
28
92
|
end
|
29
|
-
|
93
|
+
|
30
94
|
def puts message
|
31
95
|
counter = 0
|
32
96
|
begin
|
33
|
-
Logger.debug [:connector
|
97
|
+
Logger.debug [:connector, :puts, @uri, message]
|
34
98
|
socket.puts message + "\n"
|
35
99
|
rescue Exception
|
36
100
|
@socket = nil
|
37
101
|
(counter += 1) <= 5 ? retry : raise
|
38
102
|
end
|
39
103
|
end
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
end
|
44
|
-
|
45
|
-
protected
|
46
|
-
|
104
|
+
|
105
|
+
private
|
106
|
+
|
47
107
|
def socket
|
48
108
|
if @socket.nil? || @socket.closed?
|
49
|
-
Logger.debug [:connector,
|
50
|
-
@socket =
|
109
|
+
Logger.debug [:connector, :init, @uri]
|
110
|
+
@socket = @uri.scheme.eql?("tcp") ? TCPSocket.new(@uri) : UDPSocket.new(@uri)
|
51
111
|
end
|
52
112
|
@socket
|
53
113
|
end
|
54
|
-
|
114
|
+
|
55
115
|
end
|
56
116
|
end
|
data/lib/graphite-api/logger.rb
CHANGED
@@ -17,11 +17,9 @@ module GraphiteAPI
|
|
17
17
|
class Logger
|
18
18
|
class << self
|
19
19
|
attr_accessor :logger
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
def init(options)
|
24
|
-
self.logger = ::Logger.new options.fetch(:dev, STDOUT)
|
20
|
+
|
21
|
+
def init options
|
22
|
+
self.logger = ::Logger.new options[:dev] || STDOUT
|
25
23
|
self.logger.level= ::Logger.const_get options[:level].to_s.upcase
|
26
24
|
end
|
27
25
|
|
@@ -16,34 +16,30 @@
|
|
16
16
|
# slice send to graphite in X seconds slices (default is 60)
|
17
17
|
# log_level info
|
18
18
|
# -----------------------------------------------------
|
19
|
-
|
20
19
|
require 'eventmachine'
|
21
20
|
require 'socket'
|
22
|
-
require File.expand_path '../utils', __FILE__
|
23
21
|
|
24
22
|
module GraphiteAPI
|
25
23
|
class Middleware < EventMachine::Connection
|
26
24
|
|
27
|
-
include Utils
|
28
|
-
|
29
25
|
def initialize buffer
|
30
26
|
@buffer = buffer and super
|
31
27
|
end
|
32
28
|
|
33
|
-
|
29
|
+
attr_reader :buffer, :client_id
|
34
30
|
|
35
31
|
def post_init
|
36
32
|
@client_id = peername
|
37
|
-
debug [:middleware, :connecting, client_id]
|
33
|
+
Logger.debug [:middleware, :connecting, client_id]
|
38
34
|
end
|
39
35
|
|
40
36
|
def receive_data data
|
41
|
-
debug [:middleware, :message, client_id, data]
|
37
|
+
Logger.debug [:middleware, :message, client_id, data]
|
42
38
|
buffer.stream data, client_id
|
43
39
|
end
|
44
40
|
|
45
41
|
def unbind
|
46
|
-
debug [:middleware, :disconnecting, client_id]
|
42
|
+
Logger.debug [:middleware, :disconnecting, client_id]
|
47
43
|
end
|
48
44
|
|
49
45
|
def peername
|
@@ -73,5 +69,5 @@ module GraphiteAPI
|
|
73
69
|
end
|
74
70
|
end
|
75
71
|
|
76
|
-
end
|
77
|
-
end
|
72
|
+
end
|
73
|
+
end
|
data/lib/graphite-api/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: graphite-api
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Eran Barak Levi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-12-
|
11
|
+
date: 2017-12-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: eventmachine
|
@@ -52,15 +52,11 @@ files:
|
|
52
52
|
- lib/graphite-api.rb
|
53
53
|
- lib/graphite-api/buffer.rb
|
54
54
|
- lib/graphite-api/cache.rb
|
55
|
-
- lib/graphite-api/cache/memory.rb
|
56
|
-
- lib/graphite-api/cli.rb
|
57
55
|
- lib/graphite-api/client.rb
|
58
56
|
- lib/graphite-api/connector.rb
|
59
57
|
- lib/graphite-api/core_ext/numeric.rb
|
60
58
|
- lib/graphite-api/logger.rb
|
61
59
|
- lib/graphite-api/middleware.rb
|
62
|
-
- lib/graphite-api/runner.rb
|
63
|
-
- lib/graphite-api/utils.rb
|
64
60
|
- lib/graphite-api/version.rb
|
65
61
|
homepage: http://www.kontera.com
|
66
62
|
licenses:
|
@@ -74,7 +70,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
74
70
|
requirements:
|
75
71
|
- - ">="
|
76
72
|
- !ruby/object:Gem::Version
|
77
|
-
version: 1.
|
73
|
+
version: 1.9.3
|
78
74
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
79
75
|
requirements:
|
80
76
|
- - ">="
|
@@ -1,36 +0,0 @@
|
|
1
|
-
module GraphiteAPI
|
2
|
-
module Cache
|
3
|
-
class Memory
|
4
|
-
include Utils
|
5
|
-
|
6
|
-
def initialize options
|
7
|
-
Zscheduler.every(120) { clean(options[:cache]) }
|
8
|
-
end
|
9
|
-
|
10
|
-
def get time, key
|
11
|
-
cache[time.to_i][key]
|
12
|
-
end
|
13
|
-
|
14
|
-
def set time, key, value
|
15
|
-
cache[time.to_i][key] = value.to_f
|
16
|
-
end
|
17
|
-
|
18
|
-
def incr time, key, value
|
19
|
-
set(time, key, value.to_f + get(time, key))
|
20
|
-
end
|
21
|
-
|
22
|
-
private
|
23
|
-
|
24
|
-
def cache
|
25
|
-
@cache ||= nested_zero_hash
|
26
|
-
end
|
27
|
-
|
28
|
-
def clean max_age
|
29
|
-
debug [:MemoryCache, :before_clean, cache]
|
30
|
-
cache.delete_if {|t,k| Time.now.to_i - t > max_age }
|
31
|
-
debug [:MemoryCache, :after_clean, cache]
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
data/lib/graphite-api/cli.rb
DELETED
@@ -1,63 +0,0 @@
|
|
1
|
-
require 'optparse'
|
2
|
-
|
3
|
-
module GraphiteAPI
|
4
|
-
class CLI
|
5
|
-
|
6
|
-
def self.parse(argv, options)
|
7
|
-
OptionParser.new do |opts|
|
8
|
-
opts.banner = "GraphiteAPI Middleware Server"
|
9
|
-
opts.define_head "Usage: graphite-middleware [options]"
|
10
|
-
opts.define_head ""
|
11
|
-
|
12
|
-
opts.on("-g", "--graphite HOST:PORT","graphite host, in HOST:PORT format (can be specified multiple times)") do |graphite|
|
13
|
-
options[:backends] << Utils::expand_host(graphite)
|
14
|
-
end
|
15
|
-
|
16
|
-
opts.on("-p", "--port PORT","listening port (default #{options[:port]})") do |port|
|
17
|
-
options[:port] = port
|
18
|
-
end
|
19
|
-
|
20
|
-
opts.on("-l", "--log-file FILE","log file") do |file|
|
21
|
-
options[:log_file] = File::expand_path(file)
|
22
|
-
end
|
23
|
-
|
24
|
-
opts.on("-L", "--log-level LEVEL","log level (default warn)") do |level|
|
25
|
-
options[:log_level] = level
|
26
|
-
end
|
27
|
-
|
28
|
-
opts.on("-P", "--pid-file FILE","pid file (default #{options[:pid]})") do |pid_file|
|
29
|
-
options[:pid] = pid_file
|
30
|
-
end
|
31
|
-
|
32
|
-
opts.on("-d", "--daemonize","run in background") do
|
33
|
-
options[:daemonize] = true
|
34
|
-
end
|
35
|
-
|
36
|
-
opts.on("-i", "--interval INT","report every X seconds (default #{options[:interval]})") do |x_seconds|
|
37
|
-
options[:interval] = x_seconds.to_i if x_seconds.to_i > 0
|
38
|
-
end
|
39
|
-
|
40
|
-
opts.on("-s", "--slice SECONDS","send to graphite in X seconds slices (default #{options[:slice]})") do |slice|
|
41
|
-
options[:slice] = slice.to_i if slice.to_i >= 0
|
42
|
-
end
|
43
|
-
|
44
|
-
opts.on("-r", "--reanimation HOURS","reanimate records that are younger than X hours, please see README") do |exp|
|
45
|
-
(options[:cache] = exp.to_i * 3600) if exp.to_i > 0
|
46
|
-
end
|
47
|
-
|
48
|
-
opts.on("-v", "--version","Show version and exit") do |exp|
|
49
|
-
puts "Version #{GraphiteAPI.version}"
|
50
|
-
exit
|
51
|
-
end
|
52
|
-
|
53
|
-
opts.separator ""
|
54
|
-
opts.separator "More Info @ https://github.com/kontera-technologies/graphite-api"
|
55
|
-
opts.separator ""
|
56
|
-
|
57
|
-
opts.define_tail ""
|
58
|
-
opts.define_tail ""
|
59
|
-
end.parse! argv
|
60
|
-
options
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
data/lib/graphite-api/runner.rb
DELETED
@@ -1,52 +0,0 @@
|
|
1
|
-
require 'optparse'
|
2
|
-
|
3
|
-
module GraphiteAPI
|
4
|
-
class Runner
|
5
|
-
include Utils
|
6
|
-
|
7
|
-
def initialize argv
|
8
|
-
CLI.parse argv, options
|
9
|
-
validate_options
|
10
|
-
end
|
11
|
-
|
12
|
-
def run
|
13
|
-
Logger.init Hash[[:std,:level].zip options.values_at(:log_file, :log_level) ]
|
14
|
-
options[:daemonize] ? daemonize(options[:pid], &method(:run!)) : run!
|
15
|
-
end
|
16
|
-
|
17
|
-
private
|
18
|
-
|
19
|
-
def daemonize pid, &block
|
20
|
-
block_given? or raise ArgumentError.new "the block is missing..."
|
21
|
-
|
22
|
-
fork do
|
23
|
-
Process.setsid
|
24
|
-
exit if fork
|
25
|
-
Dir.chdir '/tmp'
|
26
|
-
STDIN.reopen('/dev/null')
|
27
|
-
STDOUT.reopen('/dev/null','a')
|
28
|
-
STDERR.reopen('/dev/null','a')
|
29
|
-
File.open(pid,'w') { |f| f.write(Process.pid) } rescue nil
|
30
|
-
block.call
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def run!
|
35
|
-
begin
|
36
|
-
Middleware.start options
|
37
|
-
rescue Interrupt
|
38
|
-
Logger.info "Shutting down..."
|
39
|
-
Zscheduler.stop
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
def options
|
44
|
-
@options ||= Utils.default_middleware_options
|
45
|
-
end
|
46
|
-
|
47
|
-
def validate_options
|
48
|
-
abort "You must specify at least one graphite host" if options[:backends].empty?
|
49
|
-
end
|
50
|
-
|
51
|
-
end
|
52
|
-
end
|
data/lib/graphite-api/utils.rb
DELETED
@@ -1,70 +0,0 @@
|
|
1
|
-
require 'forwardable'
|
2
|
-
require 'uri'
|
3
|
-
|
4
|
-
module GraphiteAPI
|
5
|
-
module Utils
|
6
|
-
|
7
|
-
def self.included base
|
8
|
-
base.extend ClassUtils
|
9
|
-
base.extend Forwardable
|
10
|
-
base.__send__ :include, LoggerUtils
|
11
|
-
end
|
12
|
-
|
13
|
-
module LoggerUtils
|
14
|
-
[:info,:error,:warn,:debug].each do |m|
|
15
|
-
define_method m do |*args,&block|
|
16
|
-
Logger.send m, *args, &block
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
module ClassUtils
|
22
|
-
def private_reader *args
|
23
|
-
attr_reader *args
|
24
|
-
private *args
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
def normalize_time time, slice
|
29
|
-
slice = 60 if slice.nil?
|
30
|
-
((time || Time.now).to_i / slice * slice).to_i
|
31
|
-
end
|
32
|
-
|
33
|
-
def nested_zero_hash
|
34
|
-
Hash.new {|h,k| h[k] = Hash.new {|h,k| h[k] = 0} }
|
35
|
-
end
|
36
|
-
|
37
|
-
module_function
|
38
|
-
|
39
|
-
def expand_host host
|
40
|
-
if host =~ /:\/\//
|
41
|
-
uri = URI.parse host
|
42
|
-
[ uri.host, uri.port || default_options[:port] ]
|
43
|
-
else
|
44
|
-
host, port = host.split(":")
|
45
|
-
port = port.nil? ? default_options[:port] : port.to_i
|
46
|
-
[ host, port]
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def default_options
|
51
|
-
{
|
52
|
-
:backends => [],
|
53
|
-
:cleaner_interval => 43200,
|
54
|
-
:port => 2003,
|
55
|
-
:log_level => :info,
|
56
|
-
:cache => nil,
|
57
|
-
:host => "localhost",
|
58
|
-
:prefix => [],
|
59
|
-
:interval => 0,
|
60
|
-
:slice => 60,
|
61
|
-
:pid => "/tmp/graphite-middleware.pid"
|
62
|
-
}
|
63
|
-
end
|
64
|
-
|
65
|
-
def default_middleware_options
|
66
|
-
default_options.merge(:interval => 60)
|
67
|
-
end
|
68
|
-
|
69
|
-
end
|
70
|
-
end
|