graphite-api 0.0.0.25 → 0.0.1

data/README.md

@@ -10,8 +10,8 @@ Graphite client and utilities for ruby
 * Utilities like scheduling and caching.
 
 ## Features
-* Multiple Graphite servers support - GraphiteAPI-Middleware supports sending aggregate data to multiple graphite servers, useful for large data centers and backup purposes
-* Reanimation mode - support cases which the same keys (with the same timestamps) can be received simultaneously and asynchronous from multiple input sources, in that cases GraphiteAPI-Middleware will "reanimate" old records that were already sent to Graphite server, and will send the sum of the reanimate record value + value of the just received record, to graphite server; this new summed record should override the key with the new value on Graphite database.
+* Multiple Graphite Servers Support - GraphiteAPI-Middleware supports sending aggregated data to multiple graphite servers, useful for large data centers and backup purposes
+* Reanimation mode - support cases which the same keys (same timestamps as well) can be received simultaneously and asynchronously from multiple input sources, in these cases GraphiteAPI-Middleware will "reanimate" old records (records that were already sent to Graphite server), and will send the sum of the reanimated record value + the value of the record that was just received to the graphite server; this new summed record should override the key with the new value on Graphite database.
 
 ## Client Usage
 ```ruby
@@ -78,10 +78,10 @@ cd graphite-api
 rake install
 ```
 
-# TODO:
+## TODO:
 * Documentation
-* trap signals and shutdown server
-* More tests
+* Use Redis
+* Multiple backends via client as well
 
 ## Bugs
 

data/lib/graphite-api.rb

@@ -3,7 +3,7 @@ module GraphiteAPI
 
   autoload :Version,         "#{ROOT}/graphite-api/version"
   autoload :Client,          "#{ROOT}/graphite-api/client"
-  autoload :Scheduler,       "#{ROOT}/graphite-api/scheduler"
+  autoload :Reactor,         "#{ROOT}/graphite-api/reactor"
   autoload :Connector,       "#{ROOT}/graphite-api/connector"
   autoload :Middleware,      "#{ROOT}/graphite-api/middleware"
   autoload :Runner,          "#{ROOT}/graphite-api/runner"

data/lib/graphite-api/buffer.rb

@@ -18,13 +18,14 @@
 # -----------------------------------------------------
 module GraphiteAPI
   class Buffer
-
+    include Utils
+    
     attr_reader :options,:keys_to_send,:reanimation_mode, :streamer_buff
 
-    CLOSING_STREAM_CHAR = "\n"                    # end of message - when streaming to buffer obj
-    IGNORING_CHARS      = "\r"                    # remove these chars from message
-    FLOATS_ROUND_BY = 2                           # round(x) after joining floats 
-    VALID_RECORD = /^[\w|\.]+ \d+(?:\.|\d)* \d+$/ # how a valid record should look like
+    CLOSING_STREAM_CHAR = "\n"                     # end of message - when streaming to buffer obj
+    IGNORING_CHARS      = "\r"                     # skip these chars when parsing new message
+    FLOATS_ROUND_BY = 2                            # round(x) after summing floats 
+    VALID_MESSAGE = /^[\w|\.]+ \d+(?:\.|\d)* \d+$/ # how a valid message should look like
     
     def initialize options
       @options = options
@@ -35,53 +36,55 @@ module GraphiteAPI
     end
 
     def push hash
-      Logger.debug [:buffer,:add,hash]
+      debug [:buffer,:add,hash]
       time = Utils::normalize_time(hash[:time],options[:slice])
       hash[:metric].each { |k,v| cache_set(time,k,v) }
     end
+
     alias :<< :push
     
-    def stream data, client_id = nil
-      data.each_char do |char|
+    def stream message, client_id = nil
+      message.each_char do |char|
         next if invalid_char? char
-        streamer_buff[client_id] += char 
-        if char == CLOSING_STREAM_CHAR
+        streamer_buff[client_id] << char 
+        
+        if closed_stream? streamer_buff[client_id]
           push build_metric(*streamer_buff[client_id].split) if valid streamer_buff[client_id]
           streamer_buff.delete client_id
         end
+        
       end
     end
     
-    def pull as = nil
-      Array.new.tap do |obj|
-        keys_to_send.each { |t, k| k.each { |o| obj.push cache_get(t, o, as) } }
-        clear
+    def pull(as = nil)
+      [].tap do |data|
+        keys_to_send.each { |t, k| k.each { |o| data.push cache_get(t, o, as) } } and clear
       end
     end
     
-    def empty?
-      buffer_cache.empty?
-    end
-
-    def got_new_records?
+    def new_records?
       !keys_to_send.empty?
     end
     
-    def size
-      buffer_cache.values.map {|o| o.values}.flatten.size
-    end # TODO: make it less painful
-
     private
-    def invalid_char?(char)
-      IGNORING_CHARS.include?(char)
+    def closed_stream? string
+      string[-1,1] == CLOSING_STREAM_CHAR
+    end
+    
+    def invalid_char? char
+      IGNORING_CHARS.include? char
     end
     
-    def cache_set(time, key, value)
-      buffer_cache[time][key] = (buffer_cache[time][key] + value.to_f).round(FLOATS_ROUND_BY)
+    def cache_set time, key, value
+      buffer_cache[time][key] = sum(buffer_cache[time][key],value.to_f)
       keys_to_send[time].push(key) unless keys_to_send[time].include?(key)
     end
     
-    def cache_get(time, key, as)
+    def sum float1, float2
+      (float1 + float2).round FLOATS_ROUND_BY
+    end
+        
+    def cache_get time, key, as
       metric = [prefix + key,buffer_cache[time][key],time]
       as == :string ? metric.join(" ") : metric
     end
@@ -95,8 +98,8 @@ module GraphiteAPI
       buffer_cache.clear unless reanimation_mode
     end
     
-    def valid data
-      data =~ /^[\w|\.]+ \d+(?:\.|\d)* \d+$/
+    def valid message
+      message =~ VALID_MESSAGE
     end
     
     def prefix
@@ -112,15 +115,11 @@ module GraphiteAPI
     end
 
     def clean age
-      [buffer_cache,keys_to_send].each {|o| o.delete_if {|t,k| now - t > age}}
-    end
-    
-    def now
-      Time.now.to_i
+      [buffer_cache,keys_to_send].each {|o| o.delete_if {|t,k| Time.now.to_i - t > age}}
     end
     
     def start_cleaner
-      Scheduler.every(options[:cleaner_interval]) { clean(options[:reanimation_exp]) }
+      Reactor::every(options[:cleaner_interval]) { clean(options[:reanimation_exp]) }
     end
 
   end

data/lib/graphite-api/cli.rb

@@ -43,6 +43,11 @@ module GraphiteAPI
         opts.on("-r", "--reanimation HOURS","reanimate records that are younger than X hours, please see README") do |exp|
           (options[:reanimation_exp] = exp.to_i * 3600) if exp.to_i > 0
         end
+        
+        opts.on("-v", "--version","Show version and exit") do |exp|
+          puts "Version #{GraphiteAPI.version}" 
+          exit
+        end
 
         opts.separator ""
         opts.separator "More Info @ https://github.com/kontera-technologies/graphite-api"

data/lib/graphite-api/client.rb

@@ -36,41 +36,48 @@ module GraphiteAPI
     attr_reader :options,:buffer,:connectors
     
     def initialize opt
-      @options    = build_options opt.clone
+      @options    = build_options(validate(opt.clone))
       @buffer     = GraphiteAPI::Buffer.new(options)
       @connectors = GraphiteAPI::ConnectorGroup.new(options)
       start_scheduler
     end
 
     def add_metrics(m,time = Time.now)
-      buffer << {:metric => m, :time => time}
+      buffer.push(:metric => m, :time => time)
     end
 
     def join
-      sleep while buffer.got_new_records?
+      sleep 0.1 while buffer.new_records?
     end
     
     def stop
-      Scheduler.stop
+      Reactor::stop
     end
     
     def every(frequency,&block)
-      Scheduler.every(frequency,&block)
+      Reactor::every(frequency,&block)
     end
 
     protected
     
     def start_scheduler
-      Scheduler.every(options[:interval]) { send_metrics }
+      Reactor::every(options[:interval]) { send_metrics }
     end
     
     def send_metrics
-      connectors.publish buffer.pull(:string)
+      EventMachine::defer(
+        Proc.new { buffer.pull(:string) },
+        Proc.new { |data| connectors.publish(data) }
+      ) if buffer.new_records?
+    end
+    
+    def validate opt
+      raise ArgumentError.new ":graphite must be specified" if opt[:graphite].nil?
+      opt
     end
     
     def build_options opt
       default_options.tap do |options_hash|
-        options_hash[:backends] << expand_host(opt.delete(:host)) if opt.has_key? :host
         options_hash[:backends] << expand_host(opt.delete(:graphite))
         options_hash.merge! opt
         options_hash.freeze
@@ -78,4 +85,4 @@ module GraphiteAPI
     end
     
   end
-end
+end

data/lib/graphite-api/connector_group.rb

@@ -1,6 +1,7 @@
 module GraphiteAPI
   class ConnectorGroup
-
+    include Utils
+    
     attr_reader :options, :connectors
 
     def initialize options
@@ -9,7 +10,7 @@ module GraphiteAPI
     end
 
     def publish messages
-      Logger.debug [:connector_group,:publish,messages.size, @connectors]
+      debug [:connector_group,:publish,messages.size, @connectors]
       Array(messages).each { |msg| connectors.map {|c| c.puts msg} }
     end
     

data/lib/graphite-api/logger.rb

@@ -6,10 +6,20 @@
 #     Graphite::Logger.info "shuki tuki"
 #     Graphite::Logger.debug "hihi"
 # -----------------------------------------------------
+require 'logger'
+
 module GraphiteAPI
   class Logger
     class << self
       attr_accessor :logger
+      
+      # :level => :debug
+      # :std => out|err|file-name
+      def init(options)
+        self.logger = ::Logger.new(options[:std] || STDOUT)
+        self.logger.level= eval "::Logger::#{options[:level].to_s.upcase}"
+      end
+      
       def method_missing(m,*args,&block)
         if logger.respond_to? m then logger.send(m,*args,&block) end
       end      

data/lib/graphite-api/middleware.rb

@@ -19,53 +19,54 @@
 
 require 'rubygems'
 require 'eventmachine'
-require 'logger'
 require 'socket'
 
 module GraphiteAPI
   class Middleware < EventMachine::Connection
-
+    include GraphiteAPI::Utils
+    
     attr_reader :logger,:buffer,:client_id
 
-    def initialize logger, buffer
-      @logger = logger
+    def initialize buffer
       @buffer = buffer
       super
     end
 
     def post_init
       @client_id = Socket.unpack_sockaddr_in(get_peername).reverse.join(":")
-      logger.debug [:middleware,:connecting,client_id]
+      debug [:middleware,:connecting,client_id]
     end
 
     def receive_data data
-      logger.debug [:middleware,:message,client_id,data]
+      debug [:middleware,:message,client_id,data]
       buffer.stream data, client_id
     end
 
     def unbind
-      logger.debug [:middleware,:disconnecting,client_id]
+      debug [:middleware,:disconnecting,client_id]
     end
 
     def self.start options
       EventMachine.run do
-        # Resources
-        GraphiteAPI::Logger.logger = ::Logger.new(options[:log_file] || STDOUT)
-        GraphiteAPI::Logger.level = eval "::Logger::#{options[:log_level].to_s.upcase}"
-        # TODO: move logger logic to runner
-        
-        buffer    = GraphiteAPI::Buffer.new(options)
+        buffer = GraphiteAPI::Buffer.new(options)
         connectors = GraphiteAPI::ConnectorGroup.new(options)
         
         # Starting server
-        EventMachine.start_server('0.0.0.0',options[:port],self,GraphiteAPI::Logger.logger,buffer)
+        EventMachine.start_server('0.0.0.0',options[:port],self,buffer)
         GraphiteAPI::Logger.info "Server running on port #{options[:port]}"
         
         # Send metrics to graphite every X seconds
-        GraphiteAPI::Scheduler.every( options[:interval] ) do
-          connectors.publish buffer.pull(:string) if buffer.got_new_records?
+        GraphiteAPI::Reactor::every( options[:interval] ) do
+          EventMachine::defer(
+            Proc.new { buffer.pull(:string) },
+            Proc.new { |data| connectors.publish(data) }
+          ) if buffer.new_records?        
         end # every 
         
+        GraphiteAPI::Reactor::add_shutdown_hook do           
+          connectors.publish buffer.pull(:string)
+        end
+        
       end # run 
     end # start
   end # Middleware

data/lib/graphite-api/{scheduler.rb → reactor.rb}

@@ -2,9 +2,8 @@ require 'rubygems'
 require 'eventmachine'
 
 module GraphiteAPI
-  class Scheduler
+  class Reactor
     @@wrapper = nil
-    @@timers  = []
     
     class << self
       def every(frequency,&block)
@@ -14,15 +13,21 @@ module GraphiteAPI
       end
 
       def stop
-        timers.map {|t| t.cancel}
+        timers.each(&:cancel)
+        shutdown_hooks.each(&:call)
         wrapper and EventMachine.stop
       end
 
       def join
         wrapper and wrapper.join
       end
-
+      
+      def add_shutdown_hook(&block)
+        shutdown_hooks << block
+      end
+      
       private
+      
       def start_reactor
         @@wrapper = Thread.new { EventMachine.run }
         wrapper.abort_on_exception = true
@@ -33,8 +38,17 @@ module GraphiteAPI
         EventMachine.reactor_running?
       end
       
-      def wrapper;@@wrapper end
-      def timers; @@timers end
+      def wrapper 
+        @@wrapper
+      end
+      
+      def timers
+        @@timers ||= []
+      end
+      
+      def shutdown_hooks 
+        @@shutdown_hooks ||= []
+      end
       
     end
     

data/lib/graphite-api/runner.rb

@@ -2,7 +2,7 @@ require 'optparse'
 
 module GraphiteAPI
   class Runner
-
+        
     def initialize(argv)
       GraphiteAPI::CLI::parse(argv,options)
       validate_options
@@ -15,7 +15,13 @@ module GraphiteAPI
     private
     
     def run!
-       Middleware::start options
+      GraphiteAPI::Logger.init(:std => options[:log_file], :level => options[:log_level])
+      begin
+        Middleware::start options
+      rescue Interrupt
+        GraphiteAPI::Logger.info "Shutting down..."
+        GraphiteAPI::Reactor::stop
+      end
     end
 
     def options

data/lib/graphite-api/utils.rb

@@ -1,6 +1,12 @@
 module GraphiteAPI
   module Utils
     
+    [:info,:error,:warn,:debug].each do |m|
+      define_method(m) do |*args,&block|
+        Logger.send(m,*args,&block)
+      end
+    end    
+    
     module_function
 
     def normalize_time(time,slice = 60)
@@ -40,8 +46,7 @@ module GraphiteAPI
         File.open(pid,'w') { |f| f.write(Process.pid) } rescue
         yield
       end
-
     end
-
+   
   end
 end

data/lib/graphite-api/version.rb

@@ -1,5 +1,5 @@
 module GraphiteAPI
   class Version
-    VERSION = "0.0.0.25"
+    VERSION = "0.0.1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: graphite-api
 version: !ruby/object:Gem::Version
-  version: 0.0.0.25
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-22 00:00:00.000000000 Z
+date: 2012-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &70098245796060 !ruby/object:Gem::Requirement
+  requirement: &70258166114060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: 0.3.3
   type: :runtime
   prerelease: false
-  version_requirements: *70098245796060
+  version_requirements: *70258166114060
 description: Graphite API - A Simple ruby client, aggregator daemon and API tools
 email: eran@kontera.com
 executables:
@@ -39,8 +39,8 @@ files:
 - lib/graphite-api/connector_group.rb
 - lib/graphite-api/logger.rb
 - lib/graphite-api/middleware.rb
+- lib/graphite-api/reactor.rb
 - lib/graphite-api/runner.rb
-- lib/graphite-api/scheduler.rb
 - lib/graphite-api/utils.rb
 - lib/graphite-api/version.rb
 - lib/graphite-api.rb