graphite-api 0.0.0.25 → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +5 -5
- data/lib/graphite-api.rb +1 -1
- data/lib/graphite-api/buffer.rb +35 -36
- data/lib/graphite-api/cli.rb +5 -0
- data/lib/graphite-api/client.rb +16 -9
- data/lib/graphite-api/connector_group.rb +3 -2
- data/lib/graphite-api/logger.rb +10 -0
- data/lib/graphite-api/middleware.rb +17 -16
- data/lib/graphite-api/{scheduler.rb → reactor.rb} +20 -6
- data/lib/graphite-api/runner.rb +8 -2
- data/lib/graphite-api/utils.rb +7 -2
- data/lib/graphite-api/version.rb +1 -1
- metadata +5 -5
data/README.md
CHANGED
@@ -10,8 +10,8 @@ Graphite client and utilities for ruby
|
|
10
10
|
* Utilities like scheduling and caching.
|
11
11
|
|
12
12
|
## Features
|
13
|
-
* Multiple Graphite
|
14
|
-
* Reanimation mode - support cases which the same keys (
|
13
|
+
* Multiple Graphite Servers Support - GraphiteAPI-Middleware supports sending aggregated data to multiple graphite servers, useful for large data centers and backup purposes
|
14
|
+
* Reanimation mode - support cases which the same keys (same timestamps as well) can be received simultaneously and asynchronously from multiple input sources, in these cases GraphiteAPI-Middleware will "reanimate" old records (records that were already sent to Graphite server), and will send the sum of the reanimated record value + the value of the record that was just received to the graphite server; this new summed record should override the key with the new value on Graphite database.
|
15
15
|
|
16
16
|
## Client Usage
|
17
17
|
```ruby
|
@@ -78,10 +78,10 @@ cd graphite-api
|
|
78
78
|
rake install
|
79
79
|
```
|
80
80
|
|
81
|
-
|
81
|
+
## TODO:
|
82
82
|
* Documentation
|
83
|
-
*
|
84
|
-
*
|
83
|
+
* Use Redis
|
84
|
+
* Multiple backends via client as well
|
85
85
|
|
86
86
|
## Bugs
|
87
87
|
|
data/lib/graphite-api.rb
CHANGED
@@ -3,7 +3,7 @@ module GraphiteAPI
|
|
3
3
|
|
4
4
|
autoload :Version, "#{ROOT}/graphite-api/version"
|
5
5
|
autoload :Client, "#{ROOT}/graphite-api/client"
|
6
|
-
autoload :
|
6
|
+
autoload :Reactor, "#{ROOT}/graphite-api/reactor"
|
7
7
|
autoload :Connector, "#{ROOT}/graphite-api/connector"
|
8
8
|
autoload :Middleware, "#{ROOT}/graphite-api/middleware"
|
9
9
|
autoload :Runner, "#{ROOT}/graphite-api/runner"
|
data/lib/graphite-api/buffer.rb
CHANGED
@@ -18,13 +18,14 @@
|
|
18
18
|
# -----------------------------------------------------
|
19
19
|
module GraphiteAPI
|
20
20
|
class Buffer
|
21
|
-
|
21
|
+
include Utils
|
22
|
+
|
22
23
|
attr_reader :options,:keys_to_send,:reanimation_mode, :streamer_buff
|
23
24
|
|
24
|
-
CLOSING_STREAM_CHAR = "\n"
|
25
|
-
IGNORING_CHARS = "\r"
|
26
|
-
FLOATS_ROUND_BY = 2
|
27
|
-
|
25
|
+
CLOSING_STREAM_CHAR = "\n" # end of message - when streaming to buffer obj
|
26
|
+
IGNORING_CHARS = "\r" # skip these chars when parsing new message
|
27
|
+
FLOATS_ROUND_BY = 2 # round(x) after summing floats
|
28
|
+
VALID_MESSAGE = /^[\w|\.]+ \d+(?:\.|\d)* \d+$/ # how a valid message should look like
|
28
29
|
|
29
30
|
def initialize options
|
30
31
|
@options = options
|
@@ -35,53 +36,55 @@ module GraphiteAPI
|
|
35
36
|
end
|
36
37
|
|
37
38
|
def push hash
|
38
|
-
|
39
|
+
debug [:buffer,:add,hash]
|
39
40
|
time = Utils::normalize_time(hash[:time],options[:slice])
|
40
41
|
hash[:metric].each { |k,v| cache_set(time,k,v) }
|
41
42
|
end
|
43
|
+
|
42
44
|
alias :<< :push
|
43
45
|
|
44
|
-
def stream
|
45
|
-
|
46
|
+
def stream message, client_id = nil
|
47
|
+
message.each_char do |char|
|
46
48
|
next if invalid_char? char
|
47
|
-
streamer_buff[client_id]
|
48
|
-
|
49
|
+
streamer_buff[client_id] << char
|
50
|
+
|
51
|
+
if closed_stream? streamer_buff[client_id]
|
49
52
|
push build_metric(*streamer_buff[client_id].split) if valid streamer_buff[client_id]
|
50
53
|
streamer_buff.delete client_id
|
51
54
|
end
|
55
|
+
|
52
56
|
end
|
53
57
|
end
|
54
58
|
|
55
|
-
def pull
|
56
|
-
|
57
|
-
keys_to_send.each { |t, k| k.each { |o|
|
58
|
-
clear
|
59
|
+
def pull(as = nil)
|
60
|
+
[].tap do |data|
|
61
|
+
keys_to_send.each { |t, k| k.each { |o| data.push cache_get(t, o, as) } } and clear
|
59
62
|
end
|
60
63
|
end
|
61
64
|
|
62
|
-
def
|
63
|
-
buffer_cache.empty?
|
64
|
-
end
|
65
|
-
|
66
|
-
def got_new_records?
|
65
|
+
def new_records?
|
67
66
|
!keys_to_send.empty?
|
68
67
|
end
|
69
68
|
|
70
|
-
def size
|
71
|
-
buffer_cache.values.map {|o| o.values}.flatten.size
|
72
|
-
end # TODO: make it less painful
|
73
|
-
|
74
69
|
private
|
75
|
-
def
|
76
|
-
|
70
|
+
def closed_stream? string
|
71
|
+
string[-1,1] == CLOSING_STREAM_CHAR
|
72
|
+
end
|
73
|
+
|
74
|
+
def invalid_char? char
|
75
|
+
IGNORING_CHARS.include? char
|
77
76
|
end
|
78
77
|
|
79
|
-
def cache_set
|
80
|
-
buffer_cache[time][key] = (buffer_cache[time][key]
|
78
|
+
def cache_set time, key, value
|
79
|
+
buffer_cache[time][key] = sum(buffer_cache[time][key],value.to_f)
|
81
80
|
keys_to_send[time].push(key) unless keys_to_send[time].include?(key)
|
82
81
|
end
|
83
82
|
|
84
|
-
def
|
83
|
+
def sum float1, float2
|
84
|
+
(float1 + float2).round FLOATS_ROUND_BY
|
85
|
+
end
|
86
|
+
|
87
|
+
def cache_get time, key, as
|
85
88
|
metric = [prefix + key,buffer_cache[time][key],time]
|
86
89
|
as == :string ? metric.join(" ") : metric
|
87
90
|
end
|
@@ -95,8 +98,8 @@ module GraphiteAPI
|
|
95
98
|
buffer_cache.clear unless reanimation_mode
|
96
99
|
end
|
97
100
|
|
98
|
-
def valid
|
99
|
-
|
101
|
+
def valid message
|
102
|
+
message =~ VALID_MESSAGE
|
100
103
|
end
|
101
104
|
|
102
105
|
def prefix
|
@@ -112,15 +115,11 @@ module GraphiteAPI
|
|
112
115
|
end
|
113
116
|
|
114
117
|
def clean age
|
115
|
-
[buffer_cache,keys_to_send].each {|o| o.delete_if {|t,k| now - t > age}}
|
116
|
-
end
|
117
|
-
|
118
|
-
def now
|
119
|
-
Time.now.to_i
|
118
|
+
[buffer_cache,keys_to_send].each {|o| o.delete_if {|t,k| Time.now.to_i - t > age}}
|
120
119
|
end
|
121
120
|
|
122
121
|
def start_cleaner
|
123
|
-
|
122
|
+
Reactor::every(options[:cleaner_interval]) { clean(options[:reanimation_exp]) }
|
124
123
|
end
|
125
124
|
|
126
125
|
end
|
data/lib/graphite-api/cli.rb
CHANGED
@@ -43,6 +43,11 @@ module GraphiteAPI
|
|
43
43
|
opts.on("-r", "--reanimation HOURS","reanimate records that are younger than X hours, please see README") do |exp|
|
44
44
|
(options[:reanimation_exp] = exp.to_i * 3600) if exp.to_i > 0
|
45
45
|
end
|
46
|
+
|
47
|
+
opts.on("-v", "--version","Show version and exit") do |exp|
|
48
|
+
puts "Version #{GraphiteAPI.version}"
|
49
|
+
exit
|
50
|
+
end
|
46
51
|
|
47
52
|
opts.separator ""
|
48
53
|
opts.separator "More Info @ https://github.com/kontera-technologies/graphite-api"
|
data/lib/graphite-api/client.rb
CHANGED
@@ -36,41 +36,48 @@ module GraphiteAPI
|
|
36
36
|
attr_reader :options,:buffer,:connectors
|
37
37
|
|
38
38
|
def initialize opt
|
39
|
-
@options = build_options
|
39
|
+
@options = build_options(validate(opt.clone))
|
40
40
|
@buffer = GraphiteAPI::Buffer.new(options)
|
41
41
|
@connectors = GraphiteAPI::ConnectorGroup.new(options)
|
42
42
|
start_scheduler
|
43
43
|
end
|
44
44
|
|
45
45
|
def add_metrics(m,time = Time.now)
|
46
|
-
buffer
|
46
|
+
buffer.push(:metric => m, :time => time)
|
47
47
|
end
|
48
48
|
|
49
49
|
def join
|
50
|
-
sleep while buffer.
|
50
|
+
sleep 0.1 while buffer.new_records?
|
51
51
|
end
|
52
52
|
|
53
53
|
def stop
|
54
|
-
|
54
|
+
Reactor::stop
|
55
55
|
end
|
56
56
|
|
57
57
|
def every(frequency,&block)
|
58
|
-
|
58
|
+
Reactor::every(frequency,&block)
|
59
59
|
end
|
60
60
|
|
61
61
|
protected
|
62
62
|
|
63
63
|
def start_scheduler
|
64
|
-
|
64
|
+
Reactor::every(options[:interval]) { send_metrics }
|
65
65
|
end
|
66
66
|
|
67
67
|
def send_metrics
|
68
|
-
|
68
|
+
EventMachine::defer(
|
69
|
+
Proc.new { buffer.pull(:string) },
|
70
|
+
Proc.new { |data| connectors.publish(data) }
|
71
|
+
) if buffer.new_records?
|
72
|
+
end
|
73
|
+
|
74
|
+
def validate opt
|
75
|
+
raise ArgumentError.new ":graphite must be specified" if opt[:graphite].nil?
|
76
|
+
opt
|
69
77
|
end
|
70
78
|
|
71
79
|
def build_options opt
|
72
80
|
default_options.tap do |options_hash|
|
73
|
-
options_hash[:backends] << expand_host(opt.delete(:host)) if opt.has_key? :host
|
74
81
|
options_hash[:backends] << expand_host(opt.delete(:graphite))
|
75
82
|
options_hash.merge! opt
|
76
83
|
options_hash.freeze
|
@@ -78,4 +85,4 @@ module GraphiteAPI
|
|
78
85
|
end
|
79
86
|
|
80
87
|
end
|
81
|
-
end
|
88
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
module GraphiteAPI
|
2
2
|
class ConnectorGroup
|
3
|
-
|
3
|
+
include Utils
|
4
|
+
|
4
5
|
attr_reader :options, :connectors
|
5
6
|
|
6
7
|
def initialize options
|
@@ -9,7 +10,7 @@ module GraphiteAPI
|
|
9
10
|
end
|
10
11
|
|
11
12
|
def publish messages
|
12
|
-
|
13
|
+
debug [:connector_group,:publish,messages.size, @connectors]
|
13
14
|
Array(messages).each { |msg| connectors.map {|c| c.puts msg} }
|
14
15
|
end
|
15
16
|
|
data/lib/graphite-api/logger.rb
CHANGED
@@ -6,10 +6,20 @@
|
|
6
6
|
# Graphite::Logger.info "shuki tuki"
|
7
7
|
# Graphite::Logger.debug "hihi"
|
8
8
|
# -----------------------------------------------------
|
9
|
+
require 'logger'
|
10
|
+
|
9
11
|
module GraphiteAPI
|
10
12
|
class Logger
|
11
13
|
class << self
|
12
14
|
attr_accessor :logger
|
15
|
+
|
16
|
+
# :level => :debug
|
17
|
+
# :std => out|err|file-name
|
18
|
+
def init(options)
|
19
|
+
self.logger = ::Logger.new(options[:std] || STDOUT)
|
20
|
+
self.logger.level= eval "::Logger::#{options[:level].to_s.upcase}"
|
21
|
+
end
|
22
|
+
|
13
23
|
def method_missing(m,*args,&block)
|
14
24
|
if logger.respond_to? m then logger.send(m,*args,&block) end
|
15
25
|
end
|
@@ -19,53 +19,54 @@
|
|
19
19
|
|
20
20
|
require 'rubygems'
|
21
21
|
require 'eventmachine'
|
22
|
-
require 'logger'
|
23
22
|
require 'socket'
|
24
23
|
|
25
24
|
module GraphiteAPI
|
26
25
|
class Middleware < EventMachine::Connection
|
27
|
-
|
26
|
+
include GraphiteAPI::Utils
|
27
|
+
|
28
28
|
attr_reader :logger,:buffer,:client_id
|
29
29
|
|
30
|
-
def initialize
|
31
|
-
@logger = logger
|
30
|
+
def initialize buffer
|
32
31
|
@buffer = buffer
|
33
32
|
super
|
34
33
|
end
|
35
34
|
|
36
35
|
def post_init
|
37
36
|
@client_id = Socket.unpack_sockaddr_in(get_peername).reverse.join(":")
|
38
|
-
|
37
|
+
debug [:middleware,:connecting,client_id]
|
39
38
|
end
|
40
39
|
|
41
40
|
def receive_data data
|
42
|
-
|
41
|
+
debug [:middleware,:message,client_id,data]
|
43
42
|
buffer.stream data, client_id
|
44
43
|
end
|
45
44
|
|
46
45
|
def unbind
|
47
|
-
|
46
|
+
debug [:middleware,:disconnecting,client_id]
|
48
47
|
end
|
49
48
|
|
50
49
|
def self.start options
|
51
50
|
EventMachine.run do
|
52
|
-
|
53
|
-
GraphiteAPI::Logger.logger = ::Logger.new(options[:log_file] || STDOUT)
|
54
|
-
GraphiteAPI::Logger.level = eval "::Logger::#{options[:log_level].to_s.upcase}"
|
55
|
-
# TODO: move logger logic to runner
|
56
|
-
|
57
|
-
buffer = GraphiteAPI::Buffer.new(options)
|
51
|
+
buffer = GraphiteAPI::Buffer.new(options)
|
58
52
|
connectors = GraphiteAPI::ConnectorGroup.new(options)
|
59
53
|
|
60
54
|
# Starting server
|
61
|
-
EventMachine.start_server('0.0.0.0',options[:port],self,
|
55
|
+
EventMachine.start_server('0.0.0.0',options[:port],self,buffer)
|
62
56
|
GraphiteAPI::Logger.info "Server running on port #{options[:port]}"
|
63
57
|
|
64
58
|
# Send metrics to graphite every X seconds
|
65
|
-
GraphiteAPI::
|
66
|
-
|
59
|
+
GraphiteAPI::Reactor::every( options[:interval] ) do
|
60
|
+
EventMachine::defer(
|
61
|
+
Proc.new { buffer.pull(:string) },
|
62
|
+
Proc.new { |data| connectors.publish(data) }
|
63
|
+
) if buffer.new_records?
|
67
64
|
end # every
|
68
65
|
|
66
|
+
GraphiteAPI::Reactor::add_shutdown_hook do
|
67
|
+
connectors.publish buffer.pull(:string)
|
68
|
+
end
|
69
|
+
|
69
70
|
end # run
|
70
71
|
end # start
|
71
72
|
end # Middleware
|
@@ -2,9 +2,8 @@ require 'rubygems'
|
|
2
2
|
require 'eventmachine'
|
3
3
|
|
4
4
|
module GraphiteAPI
|
5
|
-
class
|
5
|
+
class Reactor
|
6
6
|
@@wrapper = nil
|
7
|
-
@@timers = []
|
8
7
|
|
9
8
|
class << self
|
10
9
|
def every(frequency,&block)
|
@@ -14,15 +13,21 @@ module GraphiteAPI
|
|
14
13
|
end
|
15
14
|
|
16
15
|
def stop
|
17
|
-
timers.
|
16
|
+
timers.each(&:cancel)
|
17
|
+
shutdown_hooks.each(&:call)
|
18
18
|
wrapper and EventMachine.stop
|
19
19
|
end
|
20
20
|
|
21
21
|
def join
|
22
22
|
wrapper and wrapper.join
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
|
+
def add_shutdown_hook(&block)
|
26
|
+
shutdown_hooks << block
|
27
|
+
end
|
28
|
+
|
25
29
|
private
|
30
|
+
|
26
31
|
def start_reactor
|
27
32
|
@@wrapper = Thread.new { EventMachine.run }
|
28
33
|
wrapper.abort_on_exception = true
|
@@ -33,8 +38,17 @@ module GraphiteAPI
|
|
33
38
|
EventMachine.reactor_running?
|
34
39
|
end
|
35
40
|
|
36
|
-
def wrapper
|
37
|
-
|
41
|
+
def wrapper
|
42
|
+
@@wrapper
|
43
|
+
end
|
44
|
+
|
45
|
+
def timers
|
46
|
+
@@timers ||= []
|
47
|
+
end
|
48
|
+
|
49
|
+
def shutdown_hooks
|
50
|
+
@@shutdown_hooks ||= []
|
51
|
+
end
|
38
52
|
|
39
53
|
end
|
40
54
|
|
data/lib/graphite-api/runner.rb
CHANGED
@@ -2,7 +2,7 @@ require 'optparse'
|
|
2
2
|
|
3
3
|
module GraphiteAPI
|
4
4
|
class Runner
|
5
|
-
|
5
|
+
|
6
6
|
def initialize(argv)
|
7
7
|
GraphiteAPI::CLI::parse(argv,options)
|
8
8
|
validate_options
|
@@ -15,7 +15,13 @@ module GraphiteAPI
|
|
15
15
|
private
|
16
16
|
|
17
17
|
def run!
|
18
|
-
|
18
|
+
GraphiteAPI::Logger.init(:std => options[:log_file], :level => options[:log_level])
|
19
|
+
begin
|
20
|
+
Middleware::start options
|
21
|
+
rescue Interrupt
|
22
|
+
GraphiteAPI::Logger.info "Shutting down..."
|
23
|
+
GraphiteAPI::Reactor::stop
|
24
|
+
end
|
19
25
|
end
|
20
26
|
|
21
27
|
def options
|
data/lib/graphite-api/utils.rb
CHANGED
@@ -1,6 +1,12 @@
|
|
1
1
|
module GraphiteAPI
|
2
2
|
module Utils
|
3
3
|
|
4
|
+
[:info,:error,:warn,:debug].each do |m|
|
5
|
+
define_method(m) do |*args,&block|
|
6
|
+
Logger.send(m,*args,&block)
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
4
10
|
module_function
|
5
11
|
|
6
12
|
def normalize_time(time,slice = 60)
|
@@ -40,8 +46,7 @@ module GraphiteAPI
|
|
40
46
|
File.open(pid,'w') { |f| f.write(Process.pid) } rescue
|
41
47
|
yield
|
42
48
|
end
|
43
|
-
|
44
49
|
end
|
45
|
-
|
50
|
+
|
46
51
|
end
|
47
52
|
end
|
data/lib/graphite-api/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: graphite-api
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-05-02 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: eventmachine
|
16
|
-
requirement: &
|
16
|
+
requirement: &70258166114060 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,7 +21,7 @@ dependencies:
|
|
21
21
|
version: 0.3.3
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70258166114060
|
25
25
|
description: Graphite API - A Simple ruby client, aggregator daemon and API tools
|
26
26
|
email: eran@kontera.com
|
27
27
|
executables:
|
@@ -39,8 +39,8 @@ files:
|
|
39
39
|
- lib/graphite-api/connector_group.rb
|
40
40
|
- lib/graphite-api/logger.rb
|
41
41
|
- lib/graphite-api/middleware.rb
|
42
|
+
- lib/graphite-api/reactor.rb
|
42
43
|
- lib/graphite-api/runner.rb
|
43
|
-
- lib/graphite-api/scheduler.rb
|
44
44
|
- lib/graphite-api/utils.rb
|
45
45
|
- lib/graphite-api/version.rb
|
46
46
|
- lib/graphite-api.rb
|