graphite-api 0.0.0.25 → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -10,8 +10,8 @@ Graphite client and utilities for ruby
10
10
  * Utilities like scheduling and caching.
11
11
 
12
12
  ## Features
13
- * Multiple Graphite servers support - GraphiteAPI-Middleware supports sending aggregate data to multiple graphite servers, useful for large data centers and backup purposes
14
- * Reanimation mode - support cases which the same keys (with the same timestamps) can be received simultaneously and asynchronous from multiple input sources, in that cases GraphiteAPI-Middleware will "reanimate" old records that were already sent to Graphite server, and will send the sum of the reanimate record value + value of the just received record, to graphite server; this new summed record should override the key with the new value on Graphite database.
13
+ * Multiple Graphite Servers Support - GraphiteAPI-Middleware supports sending aggregated data to multiple graphite servers, useful for large data centers and backup purposes
14
+ * Reanimation mode - support cases which the same keys (same timestamps as well) can be received simultaneously and asynchronously from multiple input sources, in these cases GraphiteAPI-Middleware will "reanimate" old records (records that were already sent to Graphite server), and will send the sum of the reanimated record value + the value of the record that was just received to the graphite server; this new summed record should override the key with the new value on Graphite database.
15
15
 
16
16
  ## Client Usage
17
17
  ```ruby
@@ -78,10 +78,10 @@ cd graphite-api
78
78
  rake install
79
79
  ```
80
80
 
81
- # TODO:
81
+ ## TODO:
82
82
  * Documentation
83
- * trap signals and shutdown server
84
- * More tests
83
+ * Use Redis
84
+ * Multiple backends via client as well
85
85
 
86
86
  ## Bugs
87
87
 
@@ -3,7 +3,7 @@ module GraphiteAPI
3
3
 
4
4
  autoload :Version, "#{ROOT}/graphite-api/version"
5
5
  autoload :Client, "#{ROOT}/graphite-api/client"
6
- autoload :Scheduler, "#{ROOT}/graphite-api/scheduler"
6
+ autoload :Reactor, "#{ROOT}/graphite-api/reactor"
7
7
  autoload :Connector, "#{ROOT}/graphite-api/connector"
8
8
  autoload :Middleware, "#{ROOT}/graphite-api/middleware"
9
9
  autoload :Runner, "#{ROOT}/graphite-api/runner"
@@ -18,13 +18,14 @@
18
18
  # -----------------------------------------------------
19
19
  module GraphiteAPI
20
20
  class Buffer
21
-
21
+ include Utils
22
+
22
23
  attr_reader :options,:keys_to_send,:reanimation_mode, :streamer_buff
23
24
 
24
- CLOSING_STREAM_CHAR = "\n" # end of message - when streaming to buffer obj
25
- IGNORING_CHARS = "\r" # remove these chars from message
26
- FLOATS_ROUND_BY = 2 # round(x) after joining floats
27
- VALID_RECORD = /^[\w|\.]+ \d+(?:\.|\d)* \d+$/ # how a valid record should look like
25
+ CLOSING_STREAM_CHAR = "\n" # end of message - when streaming to buffer obj
26
+ IGNORING_CHARS = "\r" # skip these chars when parsing new message
27
+ FLOATS_ROUND_BY = 2 # round(x) after summing floats
28
+ VALID_MESSAGE = /^[\w|\.]+ \d+(?:\.|\d)* \d+$/ # how a valid message should look like
28
29
 
29
30
  def initialize options
30
31
  @options = options
@@ -35,53 +36,55 @@ module GraphiteAPI
35
36
  end
36
37
 
37
38
  def push hash
38
- Logger.debug [:buffer,:add,hash]
39
+ debug [:buffer,:add,hash]
39
40
  time = Utils::normalize_time(hash[:time],options[:slice])
40
41
  hash[:metric].each { |k,v| cache_set(time,k,v) }
41
42
  end
43
+
42
44
  alias :<< :push
43
45
 
44
- def stream data, client_id = nil
45
- data.each_char do |char|
46
+ def stream message, client_id = nil
47
+ message.each_char do |char|
46
48
  next if invalid_char? char
47
- streamer_buff[client_id] += char
48
- if char == CLOSING_STREAM_CHAR
49
+ streamer_buff[client_id] << char
50
+
51
+ if closed_stream? streamer_buff[client_id]
49
52
  push build_metric(*streamer_buff[client_id].split) if valid streamer_buff[client_id]
50
53
  streamer_buff.delete client_id
51
54
  end
55
+
52
56
  end
53
57
  end
54
58
 
55
- def pull as = nil
56
- Array.new.tap do |obj|
57
- keys_to_send.each { |t, k| k.each { |o| obj.push cache_get(t, o, as) } }
58
- clear
59
+ def pull(as = nil)
60
+ [].tap do |data|
61
+ keys_to_send.each { |t, k| k.each { |o| data.push cache_get(t, o, as) } } and clear
59
62
  end
60
63
  end
61
64
 
62
- def empty?
63
- buffer_cache.empty?
64
- end
65
-
66
- def got_new_records?
65
+ def new_records?
67
66
  !keys_to_send.empty?
68
67
  end
69
68
 
70
- def size
71
- buffer_cache.values.map {|o| o.values}.flatten.size
72
- end # TODO: make it less painful
73
-
74
69
  private
75
- def invalid_char?(char)
76
- IGNORING_CHARS.include?(char)
70
+ def closed_stream? string
71
+ string[-1,1] == CLOSING_STREAM_CHAR
72
+ end
73
+
74
+ def invalid_char? char
75
+ IGNORING_CHARS.include? char
77
76
  end
78
77
 
79
- def cache_set(time, key, value)
80
- buffer_cache[time][key] = (buffer_cache[time][key] + value.to_f).round(FLOATS_ROUND_BY)
78
+ def cache_set time, key, value
79
+ buffer_cache[time][key] = sum(buffer_cache[time][key],value.to_f)
81
80
  keys_to_send[time].push(key) unless keys_to_send[time].include?(key)
82
81
  end
83
82
 
84
- def cache_get(time, key, as)
83
+ def sum float1, float2
84
+ (float1 + float2).round FLOATS_ROUND_BY
85
+ end
86
+
87
+ def cache_get time, key, as
85
88
  metric = [prefix + key,buffer_cache[time][key],time]
86
89
  as == :string ? metric.join(" ") : metric
87
90
  end
@@ -95,8 +98,8 @@ module GraphiteAPI
95
98
  buffer_cache.clear unless reanimation_mode
96
99
  end
97
100
 
98
- def valid data
99
- data =~ /^[\w|\.]+ \d+(?:\.|\d)* \d+$/
101
+ def valid message
102
+ message =~ VALID_MESSAGE
100
103
  end
101
104
 
102
105
  def prefix
@@ -112,15 +115,11 @@ module GraphiteAPI
112
115
  end
113
116
 
114
117
  def clean age
115
- [buffer_cache,keys_to_send].each {|o| o.delete_if {|t,k| now - t > age}}
116
- end
117
-
118
- def now
119
- Time.now.to_i
118
+ [buffer_cache,keys_to_send].each {|o| o.delete_if {|t,k| Time.now.to_i - t > age}}
120
119
  end
121
120
 
122
121
  def start_cleaner
123
- Scheduler.every(options[:cleaner_interval]) { clean(options[:reanimation_exp]) }
122
+ Reactor::every(options[:cleaner_interval]) { clean(options[:reanimation_exp]) }
124
123
  end
125
124
 
126
125
  end
@@ -43,6 +43,11 @@ module GraphiteAPI
43
43
  opts.on("-r", "--reanimation HOURS","reanimate records that are younger than X hours, please see README") do |exp|
44
44
  (options[:reanimation_exp] = exp.to_i * 3600) if exp.to_i > 0
45
45
  end
46
+
47
+ opts.on("-v", "--version","Show version and exit") do |exp|
48
+ puts "Version #{GraphiteAPI.version}"
49
+ exit
50
+ end
46
51
 
47
52
  opts.separator ""
48
53
  opts.separator "More Info @ https://github.com/kontera-technologies/graphite-api"
@@ -36,41 +36,48 @@ module GraphiteAPI
36
36
  attr_reader :options,:buffer,:connectors
37
37
 
38
38
  def initialize opt
39
- @options = build_options opt.clone
39
+ @options = build_options(validate(opt.clone))
40
40
  @buffer = GraphiteAPI::Buffer.new(options)
41
41
  @connectors = GraphiteAPI::ConnectorGroup.new(options)
42
42
  start_scheduler
43
43
  end
44
44
 
45
45
  def add_metrics(m,time = Time.now)
46
- buffer << {:metric => m, :time => time}
46
+ buffer.push(:metric => m, :time => time)
47
47
  end
48
48
 
49
49
  def join
50
- sleep while buffer.got_new_records?
50
+ sleep 0.1 while buffer.new_records?
51
51
  end
52
52
 
53
53
  def stop
54
- Scheduler.stop
54
+ Reactor::stop
55
55
  end
56
56
 
57
57
  def every(frequency,&block)
58
- Scheduler.every(frequency,&block)
58
+ Reactor::every(frequency,&block)
59
59
  end
60
60
 
61
61
  protected
62
62
 
63
63
  def start_scheduler
64
- Scheduler.every(options[:interval]) { send_metrics }
64
+ Reactor::every(options[:interval]) { send_metrics }
65
65
  end
66
66
 
67
67
  def send_metrics
68
- connectors.publish buffer.pull(:string)
68
+ EventMachine::defer(
69
+ Proc.new { buffer.pull(:string) },
70
+ Proc.new { |data| connectors.publish(data) }
71
+ ) if buffer.new_records?
72
+ end
73
+
74
+ def validate opt
75
+ raise ArgumentError.new ":graphite must be specified" if opt[:graphite].nil?
76
+ opt
69
77
  end
70
78
 
71
79
  def build_options opt
72
80
  default_options.tap do |options_hash|
73
- options_hash[:backends] << expand_host(opt.delete(:host)) if opt.has_key? :host
74
81
  options_hash[:backends] << expand_host(opt.delete(:graphite))
75
82
  options_hash.merge! opt
76
83
  options_hash.freeze
@@ -78,4 +85,4 @@ module GraphiteAPI
78
85
  end
79
86
 
80
87
  end
81
- end
88
+ end
@@ -1,6 +1,7 @@
1
1
  module GraphiteAPI
2
2
  class ConnectorGroup
3
-
3
+ include Utils
4
+
4
5
  attr_reader :options, :connectors
5
6
 
6
7
  def initialize options
@@ -9,7 +10,7 @@ module GraphiteAPI
9
10
  end
10
11
 
11
12
  def publish messages
12
- Logger.debug [:connector_group,:publish,messages.size, @connectors]
13
+ debug [:connector_group,:publish,messages.size, @connectors]
13
14
  Array(messages).each { |msg| connectors.map {|c| c.puts msg} }
14
15
  end
15
16
 
@@ -6,10 +6,20 @@
6
6
  # Graphite::Logger.info "shuki tuki"
7
7
  # Graphite::Logger.debug "hihi"
8
8
  # -----------------------------------------------------
9
+ require 'logger'
10
+
9
11
  module GraphiteAPI
10
12
  class Logger
11
13
  class << self
12
14
  attr_accessor :logger
15
+
16
+ # :level => :debug
17
+ # :std => out|err|file-name
18
+ def init(options)
19
+ self.logger = ::Logger.new(options[:std] || STDOUT)
20
+ self.logger.level= eval "::Logger::#{options[:level].to_s.upcase}"
21
+ end
22
+
13
23
  def method_missing(m,*args,&block)
14
24
  if logger.respond_to? m then logger.send(m,*args,&block) end
15
25
  end
@@ -19,53 +19,54 @@
19
19
 
20
20
  require 'rubygems'
21
21
  require 'eventmachine'
22
- require 'logger'
23
22
  require 'socket'
24
23
 
25
24
  module GraphiteAPI
26
25
  class Middleware < EventMachine::Connection
27
-
26
+ include GraphiteAPI::Utils
27
+
28
28
  attr_reader :logger,:buffer,:client_id
29
29
 
30
- def initialize logger, buffer
31
- @logger = logger
30
+ def initialize buffer
32
31
  @buffer = buffer
33
32
  super
34
33
  end
35
34
 
36
35
  def post_init
37
36
  @client_id = Socket.unpack_sockaddr_in(get_peername).reverse.join(":")
38
- logger.debug [:middleware,:connecting,client_id]
37
+ debug [:middleware,:connecting,client_id]
39
38
  end
40
39
 
41
40
  def receive_data data
42
- logger.debug [:middleware,:message,client_id,data]
41
+ debug [:middleware,:message,client_id,data]
43
42
  buffer.stream data, client_id
44
43
  end
45
44
 
46
45
  def unbind
47
- logger.debug [:middleware,:disconnecting,client_id]
46
+ debug [:middleware,:disconnecting,client_id]
48
47
  end
49
48
 
50
49
  def self.start options
51
50
  EventMachine.run do
52
- # Resources
53
- GraphiteAPI::Logger.logger = ::Logger.new(options[:log_file] || STDOUT)
54
- GraphiteAPI::Logger.level = eval "::Logger::#{options[:log_level].to_s.upcase}"
55
- # TODO: move logger logic to runner
56
-
57
- buffer = GraphiteAPI::Buffer.new(options)
51
+ buffer = GraphiteAPI::Buffer.new(options)
58
52
  connectors = GraphiteAPI::ConnectorGroup.new(options)
59
53
 
60
54
  # Starting server
61
- EventMachine.start_server('0.0.0.0',options[:port],self,GraphiteAPI::Logger.logger,buffer)
55
+ EventMachine.start_server('0.0.0.0',options[:port],self,buffer)
62
56
  GraphiteAPI::Logger.info "Server running on port #{options[:port]}"
63
57
 
64
58
  # Send metrics to graphite every X seconds
65
- GraphiteAPI::Scheduler.every( options[:interval] ) do
66
- connectors.publish buffer.pull(:string) if buffer.got_new_records?
59
+ GraphiteAPI::Reactor::every( options[:interval] ) do
60
+ EventMachine::defer(
61
+ Proc.new { buffer.pull(:string) },
62
+ Proc.new { |data| connectors.publish(data) }
63
+ ) if buffer.new_records?
67
64
  end # every
68
65
 
66
+ GraphiteAPI::Reactor::add_shutdown_hook do
67
+ connectors.publish buffer.pull(:string)
68
+ end
69
+
69
70
  end # run
70
71
  end # start
71
72
  end # Middleware
@@ -2,9 +2,8 @@ require 'rubygems'
2
2
  require 'eventmachine'
3
3
 
4
4
  module GraphiteAPI
5
- class Scheduler
5
+ class Reactor
6
6
  @@wrapper = nil
7
- @@timers = []
8
7
 
9
8
  class << self
10
9
  def every(frequency,&block)
@@ -14,15 +13,21 @@ module GraphiteAPI
14
13
  end
15
14
 
16
15
  def stop
17
- timers.map {|t| t.cancel}
16
+ timers.each(&:cancel)
17
+ shutdown_hooks.each(&:call)
18
18
  wrapper and EventMachine.stop
19
19
  end
20
20
 
21
21
  def join
22
22
  wrapper and wrapper.join
23
23
  end
24
-
24
+
25
+ def add_shutdown_hook(&block)
26
+ shutdown_hooks << block
27
+ end
28
+
25
29
  private
30
+
26
31
  def start_reactor
27
32
  @@wrapper = Thread.new { EventMachine.run }
28
33
  wrapper.abort_on_exception = true
@@ -33,8 +38,17 @@ module GraphiteAPI
33
38
  EventMachine.reactor_running?
34
39
  end
35
40
 
36
- def wrapper;@@wrapper end
37
- def timers; @@timers end
41
+ def wrapper
42
+ @@wrapper
43
+ end
44
+
45
+ def timers
46
+ @@timers ||= []
47
+ end
48
+
49
+ def shutdown_hooks
50
+ @@shutdown_hooks ||= []
51
+ end
38
52
 
39
53
  end
40
54
 
@@ -2,7 +2,7 @@ require 'optparse'
2
2
 
3
3
  module GraphiteAPI
4
4
  class Runner
5
-
5
+
6
6
  def initialize(argv)
7
7
  GraphiteAPI::CLI::parse(argv,options)
8
8
  validate_options
@@ -15,7 +15,13 @@ module GraphiteAPI
15
15
  private
16
16
 
17
17
  def run!
18
- Middleware::start options
18
+ GraphiteAPI::Logger.init(:std => options[:log_file], :level => options[:log_level])
19
+ begin
20
+ Middleware::start options
21
+ rescue Interrupt
22
+ GraphiteAPI::Logger.info "Shutting down..."
23
+ GraphiteAPI::Reactor::stop
24
+ end
19
25
  end
20
26
 
21
27
  def options
@@ -1,6 +1,12 @@
1
1
  module GraphiteAPI
2
2
  module Utils
3
3
 
4
+ [:info,:error,:warn,:debug].each do |m|
5
+ define_method(m) do |*args,&block|
6
+ Logger.send(m,*args,&block)
7
+ end
8
+ end
9
+
4
10
  module_function
5
11
 
6
12
  def normalize_time(time,slice = 60)
@@ -40,8 +46,7 @@ module GraphiteAPI
40
46
  File.open(pid,'w') { |f| f.write(Process.pid) } rescue
41
47
  yield
42
48
  end
43
-
44
49
  end
45
-
50
+
46
51
  end
47
52
  end
@@ -1,5 +1,5 @@
1
1
  module GraphiteAPI
2
2
  class Version
3
- VERSION = "0.0.0.25"
3
+ VERSION = "0.0.1"
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: graphite-api
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0.25
4
+ version: 0.0.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-04-22 00:00:00.000000000 Z
12
+ date: 2012-05-02 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: eventmachine
16
- requirement: &70098245796060 !ruby/object:Gem::Requirement
16
+ requirement: &70258166114060 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
21
21
  version: 0.3.3
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70098245796060
24
+ version_requirements: *70258166114060
25
25
  description: Graphite API - A Simple ruby client, aggregator daemon and API tools
26
26
  email: eran@kontera.com
27
27
  executables:
@@ -39,8 +39,8 @@ files:
39
39
  - lib/graphite-api/connector_group.rb
40
40
  - lib/graphite-api/logger.rb
41
41
  - lib/graphite-api/middleware.rb
42
+ - lib/graphite-api/reactor.rb
42
43
  - lib/graphite-api/runner.rb
43
- - lib/graphite-api/scheduler.rb
44
44
  - lib/graphite-api/utils.rb
45
45
  - lib/graphite-api/version.rb
46
46
  - lib/graphite-api.rb