graphite-api 0.0.3.beta1 → 0.0.3.beta3

data/lib/graphite-api/buffer.rb

@@ -30,9 +30,9 @@ module GraphiteAPI
     
     def initialize options
       @options = options
-      @keys_to_sync  = Hash.new { |h,k| h[k] = Set.new }
+      @keys_to_sync = Hash.new { |h,k| h[k] = Set.new }
       @streamer_buff = Hash.new {|h,k| h[k] = ""}
-      @reanimation_mode = !options[:reanimation_exp].nil?
+      @reanimation_mode = !options[:cache].nil?
       start_cleaner if reanimation_mode
     end
 
@@ -40,7 +40,7 @@ module GraphiteAPI
 
     def push hash
       debug [:buffer,:add, hash]
-      time = Utils.normalize_time(hash[:time],options[:slice])
+      time = normalize_time(hash[:time],options[:slice])
       hash[:metric].each { |k,v| cache_set(time,k,v) }
     end
 
@@ -130,7 +130,7 @@ module GraphiteAPI
     end
     
     def start_cleaner
-      Reactor::every(options[:cleaner_interval]) { clean(options[:reanimation_exp]) }
+      Reactor::every(options[:cleaner_interval]) { clean(options[:cache]) }
     end
 
   end

data/lib/graphite-api/cache/memory.rb

@@ -1,9 +1,10 @@
 module GraphiteAPI
   module Cache
     class Memory
-
+      include Utils
+      
       def initialize options
-        Reactor.every(60) { clean(options[:age]) }
+        Reactor.every(120) { clean(options[:cache]) }
       end
       
       def get time, key
@@ -14,8 +15,8 @@ module GraphiteAPI
         cache[time][key] = value.to_f 
       end
       
-      def incr *args
-        set(value.to_f + get(*args))
+      def incr time, key, value
+        set(time, key, value.to_f + get(time, key))
       end
       
       private
@@ -25,7 +26,9 @@ module GraphiteAPI
       end
       
       def clean age
+        debug [:MemoryCache, :before_clean, cache]
         cache.delete_if {|t,k| Time.now.to_i - t > age}
+        debug [:MemoryCache, :after_clean, cache]
       end
       
     end

data/lib/graphite-api/cli.rb

@@ -42,7 +42,7 @@ module GraphiteAPI
         end
 
         opts.on("-r", "--reanimation HOURS","reanimate records that are younger than X hours, please see README") do |exp|
-          (options[:reanimation_exp] = exp.to_i * 3600) if exp.to_i > 0
+          (options[:cache] = exp.to_i * 3600) if exp.to_i > 0
         end
         
         opts.on("-v", "--version","Show version and exit") do |exp|

data/lib/graphite-api/connector.rb

@@ -24,6 +24,7 @@ module GraphiteAPI
     def puts message
       begin
         Logger.debug [:connector,:puts,[host,port].join(":"),message]
+        return
         socket.puts message + "\n"
       rescue Errno::EPIPE, Errno::EINVAL
         @socket = nil

data/lib/graphite-api/middleware.rb

@@ -57,7 +57,7 @@ module GraphiteAPI
       EventMachine.run do
         GraphiteAPI::Logger.info "Server running on port #{options[:port]}"
         
-        buffer = GraphiteAPI::Buffer.new options
+        buffer = GraphiteAPI::SafeBuffer.new options
         group  = GraphiteAPI::ConnectorGroup.new options
         
         # Starting server

data/lib/graphite-api/safe_buffer.rb

@@ -1,3 +1,22 @@
+# -----------------------------------------------------
+# Buffer Object
+# Handle Socket & Client data streams
+# -----------------------------------------------------
+# Usage:
+#     buff = GraphiteAPI::SafeBuffer.new(GraphiteAPI::Utils.default_options)
+#     buff << {:metric => {"load_avg" => 10},:time => Time.now}
+#     buff << {:metric => {"load_avg" => 30},:time => Time.now}
+#     buff.stream "mem.usage 1"
+#     buff.stream "90 1326842563\n"
+#     buff.stream "shuki.tuki 999 1326842563\n"
+#     buff.pull.each {|o| p o} 
+#
+# Produce:
+#    ["load_avg", 40.0, 1326881160]
+#    ["mem.usage", 190.0, 1326842520]
+#    ["shuki.tuki", 999.0, 1326842520]
+# -----------------------------------------------------
+
 require 'thread'
 require 'set'
 
@@ -5,30 +24,24 @@ module GraphiteAPI
   class SafeBuffer
     include Utils
     
+    CHARS_TO_BE_IGNORED = ["\r"]
+    END_OF_STREAM = "\n"
+    VALID_MESSAGE = /^[\w|\.]+ \d+(?:\.|\d)* \d+$/
+    
     def initialize options
       @options = options
       @queue = Queue.new
       @streamer = Hash.new {|h,k| h[k] = ""}
 
-      if options[:reanimation_exp]
-        @cache = Cache::Memory.new options[:reanimation_exp]
-      end      
+      if options[:cache]
+        @cache = Cache::Memory.new options
+      end
     end
     
     private_reader :queue, :options, :streamer, :cache
     
-    # {:metric => {'a' => 10},:time => today}
-    def push hash
-      debug [:buffer,:add, hash]
-      time = Utils.normalize_time(hash[:time],options[:slice])
-      hash[:metric].each { |k,v| queue.push [time,k,v] }
-    end
-    
-    alias_method :<<, :push
-
     # this method isn't thread safe
-    # if you are running with multiple threads
-    # use #push instead
+    # use #push for multiple threads support
     def stream message, client_id = nil
       message.gsub(/\t/,' ').each_char do |char|
         next if invalid_char? char
@@ -43,12 +56,27 @@ module GraphiteAPI
       end
     end
     
+    # Add records to buffer
+    # push({:metric => {'a' => 10},:time => Time.now})
+    def push obj
+      debug [:buffer,:add, obj]
+      queue.push obj
+      nil
+    end    
+
+    alias_method :<<, :push
+
     def pull format = nil
-      data = Hash.new { |h,k| h[k] = Hash.new { |h,k| h[k] = 0} }
+      data = Hash.new {|h,k| h[k] = Hash.new {|h,k| h[k] = 0} }
+
+      counter = 0
       while new_records?
-        time, key, value = queue.pop
-        data[time][key] += value.to_f
+        break if ( counter += 1 ) > 10_000
+        hash = queue.pop
+        time = normalize_time(hash[:time],options[:slice])
+        hash[:metric].each { |k,v| data[time][k] += v.to_f }
       end
+      
       data.map do |time, hash|
         hash.map do |key, value|
           value = cache.incr(time,key,value) if cache
@@ -62,6 +90,10 @@ module GraphiteAPI
       !queue.empty?
     end
     
+    def inspect
+      "#<GraphiteAPI::SafeBuffer:#{object_id} @quque#size=#{queue.size} @streamer=#{streamer.inspect}>"
+    end
+    
     private
     
     def stream_message_to_obj message
@@ -70,19 +102,23 @@ module GraphiteAPI
     end
     
     def invalid_char? char
-      ["\r"].include? char
+      CHARS_TO_BE_IGNORED.include? char
     end
     
     def closed_stream? string
-      string[-1,1] == "\n"
+      string[-1,1] == END_OF_STREAM
     end
     
     def valid_stream_message message
-      message =~ /^[\w|\.]+ \d+(?:\.|\d)* \d+$/
+      message =~ VALID_MESSAGE
     end
-    
+
     def prefix
-      @prefix ||= options[:prefix].empty? ? '' : Array(options[:prefix]).join('.') << '.'
+      @prefix ||= if options[:prefix] and !options[:prefix].empty?
+        Array(options[:prefix]).join('.') << '.'
+      else
+        ""
+      end
     end
         
   end

data/lib/graphite-api/utils.rb

@@ -23,13 +23,14 @@ module GraphiteAPI
         private     *args
       end
     end
-        
-    module_function
-
-    def normalize_time time, slice = 60
+    
+    def normalize_time time, slice
+      slice = 60 if slice.nil?
       ((time || Time.now).to_i / slice * slice).to_i
     end
  
+    module_function
+ 
     def expand_host host
       host,port = host.split(":")
       port = port.nil? ? default_options[:port] : port.to_i
@@ -42,7 +43,7 @@ module GraphiteAPI
         :cleaner_interval => 43200,
         :port => 2003,
         :log_level => :info,
-        :reanimation_exp => nil,
+        :cache => nil,
         :host => "localhost",
         :prefix => [],
         :interval => 60,

data/lib/graphite-api/version.rb

@@ -1,5 +1,5 @@
 module GraphiteAPI
   class Version
-    VERSION = "0.0.3.beta1"
+    VERSION = "0.0.3.beta3"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: graphite-api
 version: !ruby/object:Gem::Version
-  version: 0.0.3.beta1
+  version: 0.0.3.beta3
   prerelease: 6
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-04 00:00:00.000000000 Z
+date: 2013-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine