graphite-api 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8d03b673ba89014ffbe0bb86c4455350c51aaba5
-  data.tar.gz: 0b257ccfefc1ccb43fd8a1ce76c51e962f55a6a7
+SHA256:
+  metadata.gz: 76010192ae0104397b2f2f069f03ee6f5895f42473f8bab79068ce7808104a58
+  data.tar.gz: da60c3a2a69ed0d1e35232d4b1e4fb90709a7d16c87d241801824b67094cf4b0
 SHA512:
-  metadata.gz: 9441471eb45d0dda1cb34d9c562534295ecb96cc0f2fcd5e8fcabfd75c52666f7818f6aeaae07476f6af2026159174015eb75667e0e4341a4bf98dd9e98f68b9
-  data.tar.gz: f00e42c5824473da725b9ec89787982f9fe3055bab4512abab23b3bf047501c6c4a21e36ca1231dacec66f4094fdb4a981ba1643566b34d2cf7419b759623b2f
+  metadata.gz: 29d966b90c0d676c604e2556a05992667ec6a083e078ae2a2a4cb75a9524ea042919585038ff18dcf1b90b7ed049c414ebefdb5dbc0389787a2976960cf534b9
+  data.tar.gz: 4c1c95a572507ecd6df22581559f615fecdefc8cb8c098707bdaac0ed2bc39f51c6dcbeaa13b3dee441a3b36a0e86cbf30d0e916676214aa0aa40aa38480edc8

data/README.md

@@ -13,17 +13,9 @@
 * **Thread-Safe** client.
 
 ## Status
-<table>
-  <tr>
-    <td> Version </td>
-    <td><a href="https://rubygems.org/gems/graphite-api"><img src=https://fury-badge.herokuapp.com/rb/graphite-api.png></a> </td>
-  </tr>
-  <tr>
-    <td> Build </td>
-    <td><a href="https://travis-ci.org/kontera-technologies/graphite-api"><img src=https://travis-ci.org/kontera-technologies/graphite-api.png?branch=master></a>
-      </td>
-  </tr>
-</table>
+[![Gem Version](https://badge.fury.io/rb/graphite-api.svg)](https://badge.fury.io/rb/graphite-api)
+[![Build Status](https://travis-ci.org/kontera-technologies/graphite-api.svg?branch=master)](https://travis-ci.org/kontera-technologies/graphite-api)
+[![Test Coverage](https://codecov.io/gh/kontera-technologies/graphite-api/branch/master/graph/badge.svg)](https://codecov.io/gh/kontera-technologies/graphite-api)
 
 ## Installation
 Install stable version
@@ -32,14 +24,6 @@ Install stable version
 gem install graphite-api
 ```
 
-Install the latest from github
-
-```
-git clone git://github.com/kontera-technologies/graphite-api.git
-cd graphite-api
-rake install
-```
-
 ## Client Usage
 
 Creating a new UDP client
@@ -61,7 +45,11 @@ options = {
   interval: 60,
 
   # Optional: set the max age in seconds for records reanimation ( default is 12 hours )
-  cache: 4 * 60 * 60
+  cache: 4 * 60 * 60,
+
+  # Optional: The default aggregation method for multiple reports in the same slice (default is :add).
+  # Possible options: :sum, :avg, :replace
+  default_aggregation_method: :avg
 }
 
 client = GraphiteAPI.new options
@@ -77,6 +65,11 @@ TCP Client with 30 seconds timeout
 client = GraphiteAPI.new graphite: "tcp://graphite.example.com:2003?timeout=30"
 ```
 
+TCP Client with custom aggregation method
+```ruby
+client = GraphiteAPI.new graphite: "tcp://graphite.example.com:2003", default_aggregation_method: :avg
+```
+
 Adding simple metrics
 ```ruby
 require 'graphite-api'
@@ -108,6 +101,24 @@ client.metrics({
 # => webServer.web01.memUsage 40 1326067060
 ```
 
+Adding metrics with custom aggregation method
+```ruby
+require 'graphite-api'
+
+client = GraphiteAPI.new( graphite: 'udp://graphite:2003' )
+
+client.metrics({
+  "webServer.web01.loadAvg"  => 10,
+  "webServer.web01.memUsage" => 40
+},Time.at(1326067060), :avg)
+client.metrics({
+  "webServer.web01.loadAvg"  => 20,
+  "webServer.web01.memUsage" => 50
+},Time.at(1326067060), :avg)
+# => webServer.web01.loadAvg  15 1326067060
+# => webServer.web01.memUsage 45 1326067060
+```
+
 Verifying connectivity
 ```ruby
 require 'graphite-api'
@@ -225,6 +236,7 @@ Usage: graphite-middleware [options]
     -i, --interval INT               report every X seconds (default 60)
     -s, --slice SECONDS              send to graphite in X seconds slices (default 60)
     -r, --reanimation HOURS          reanimate records that are younger than X hours, please see README
+    -m, --aggregation-method method  The aggregation method (sum, avg or replace) for multiple reports in the same time slice (default sum)
 
 More Info @ https://github.com/kontera-technologies/graphite-api
 ```

data/Rakefile

@@ -9,7 +9,7 @@ def message msg
   puts "*** #{msg} ***"
 end
 
-task(:test => :functional) { ENV['with_coverage'] = "true" }
+task(:test => :functional)
 
 Rake::TestTask.new do |t|
   t.libs << "tests"
@@ -18,14 +18,14 @@ end
 
 task :functional do
   some_failed = false
-  
+
   next unless ENV['SKIP_FUNC'].nil?
 
   unless RUBY_COPYRIGHT.end_with?("Matsumoto")
     puts("Functional tests are enabled only on MRI...")
     next
   end
-  
+
   message "Executing GraphiteAPI Functional Tests"
   message "( You can skip them by passing SKIP_FUNC=true )"
 
@@ -56,8 +56,3 @@ GraphiteAPI::GemSpec = eval File.read 'graphite-api.gemspec'
 Gem::PackageTask.new(GraphiteAPI::GemSpec) do |p|
   p.gem_spec = GraphiteAPI::GemSpec
 end
-
-task :install => [:gem] do
-   sh "gem install pkg/graphite-api"
-   Rake::Task['clobber_package'].execute
-end

data/bin/graphite-middleware

@@ -31,7 +31,7 @@ OptionParser.new do |opts|
     options[:pid] = pid_file
   end
 
-  opts.on("-d", "--daemonize","run in background") do 
+  opts.on("-d", "--daemonize","run in background") do
     options[:daemonize] = true
   end
 
@@ -47,8 +47,13 @@ OptionParser.new do |opts|
     (options[:cache] = exp.to_i * 3600) if exp.to_i > 0
   end
 
+  opts.on("-m", "--aggregation-method method","The aggregation method (sum, avg or replace) for multiple reports in the same time slice (default sum)") do |exp|
+    raise "Invalid aggregation method. Valid values are sum, avg or replace." unless ["sum", "avg", "replace"].include? exp
+    options[:default_aggregation_method] = exp.to_sym
+  end
+
   opts.on("-v", "--version","Show version and exit") do |exp|
-    puts "Version #{GraphiteAPI.version}" 
+    puts "Version #{GraphiteAPI.version}"
     exit
   end
 

data/lib/graphite-api/buffer.rb

@@ -3,16 +3,16 @@
 # Handle Socket & Client data streams
 # -----------------------------------------------------
 # Usage:
-#     buff = GraphiteAPI::Buffer.new(GraphiteAPI::Utils.default_options)
-#     buff << {:metric => {"load_avg" => 10},:time => Time.now}
-#     buff << {:metric => {"load_avg" => 30},:time => Time.now}
+#     buff = GraphiteAPI::Buffer.new(GraphiteAPI::Client.default_options)
+#     buff << {:metric => {"load_avg" => 10},:time => Time.now, :aggregation_method => :avg}
+#     buff << {:metric => {"load_avg" => 30},:time => Time.now, :aggregation_method => :avg}
 #     buff.stream "mem.usage 1"
 #     buff.stream "90 1326842563\n"
 #     buff.stream "shuki.tuki 999 1326842563\n"
-#     buff.pull.each {|o| p o} 
+#     buff.pull.each {|o| p o}
 #
 # Produce:
-#    ["load_avg", 40.0, 1326881160]
+#    ["load_avg", 20.0, 1326881160]
 #    ["mem.usage", 190.0, 1326842520]
 #    ["shuki.tuki", 999.0, 1326842520]
 # -----------------------------------------------------
@@ -21,38 +21,44 @@ require 'set'
 
 module GraphiteAPI
   class Buffer
-    
+
     IGNORE = ["\r"]
     END_OF_STREAM = "\n"
     VALID_MESSAGE = /^[\w|\.|-]+ \d+(?:\.|\d)* \d+$/
-    
+
+    AGGREGATORS = {
+      sum: ->(*args) { args.reduce(0) { |sum, x| sum + x } },
+      avg: ->(*args) { args.reduce(0) { |sum, x| sum + x } / [args.length, 1].max },
+      replace: ->(*args) { args.last },
+    }
+
     def initialize options
       @options = options
       @queue = Queue.new
       @streamer = Hash.new {|h,k| h[k] = ""}
-      @cache = Cache::Memory.new options if options[:cache]
+      @cache = Cache::Memory.new(options) if options[:cache]
     end
-    
+
     attr_reader :queue, :options, :streamer, :cache
-    
+
     # this method isn't thread safe
     # use #push for multiple threads support
     def stream message, client_id = nil
       message.gsub(/\t/,' ').each_char do |char|
         next if invalid_char? char
-        streamer[client_id] += char 
-        
+        streamer[client_id] += char
+
         if closed_stream? streamer[client_id]
-          if streamer[client_id] =~ VALID_MESSAGE 
+          if streamer[client_id] =~ VALID_MESSAGE
             push stream_message_to_obj streamer[client_id]
           end
           streamer.delete client_id
         end
       end
     end
-    
+
     # Add records to buffer
-    # push({:metric => {'a' => 10},:time => Time.now})
+    # push({:metric => {'a' => 10},:time => Time.now,:aggregation_method => :sum})
     def push obj
       Logger.debug [:buffer,:add, obj]
       queue.push obj
@@ -62,49 +68,67 @@ module GraphiteAPI
     alias_method :<<, :push
 
     def pull format = nil
-      data = Hash.new {|h,k| h[k] = Hash.new {|h2,k2| h2[k2] = 0}}
+      data = Hash.new { |h,time| h[time] = Hash.new { |h2,metric| h2[metric] = cache_get(time, metric) } }
+      aggregation_methods = Hash.new
 
       counter = 0
-      while new_records?
-        break if ( counter += 1 ) > 1_000_000 # TODO: fix this
-        hash = queue.pop
-        hash[:metric].each {|k,v| data[normalize_time(hash[:time],options[:slice])][k] += v.to_f}
+      while new_records? and (counter += 1) < 1_000_000
+        metrics, time, method_name = queue.pop.values_at(:metric, :time, :aggregation_method)
+
+        normalized_time = normalize_time(time, options[:slice])
+        metrics.each do |metric, value|
+          aggregation_methods[metric] = method_name || options[:default_aggregation_method]
+          data[normalized_time][metric].push value.to_f
+          cache_set(normalized_time, metric, data[normalized_time][metric])
+        end
       end
-      
-      data.map do |time, hash|
-        hash.map do |key, value|
-          value = cache.incr(time,key,value) if cache
-          results = ["#{prefix}#{key}",("%f"%value).to_f, time]
+
+      data.map do |time, metrics|
+        metrics.map do |metric, raw_values|
+          value = AGGREGATORS[aggregation_methods[metric]].call(*raw_values)
+          results = ["#{prefix}#{metric}",("%f"%value).to_f, time]
           format == :string ? results.join(" ") : results
         end
       end.flatten(1)
     end
-    
+
     def inspect
-      "#<GraphiteAPI::Buffer:%s @quque#size=%s @streamer=%s>" % 
+      "#<GraphiteAPI::Buffer:%s @quque#size=%s @streamer=%s>" %
         [ object_id, queue.size, streamer]
     end
-    
+
     def new_records?
       !queue.empty?
     end
 
     private
 
+    def cache_get time, metric
+      if cache
+        cache.get(time, metric) || []
+      else
+        []
+      end
+    end
+
+    def cache_set time, metric, value
+      cache.set(time, metric, value) if cache
+    end
+
     def normalize_time time, slice
       slice = 60 if slice.nil?
       ((time || Time.now).to_i / slice * slice).to_i
     end
-    
+
     def stream_message_to_obj message
       parts = message.split
       {:metric => { parts[0] => parts[1] },:time => Time.at(parts[2].to_i) }
     end
-    
+
     def invalid_char? char
       IGNORE.include? char
     end
-    
+
     def closed_stream? string
       string[-1,1] == END_OF_STREAM
     end
@@ -116,6 +140,6 @@ module GraphiteAPI
         ""
       end
     end
-        
+
   end
 end

data/lib/graphite-api/cache.rb

@@ -11,17 +11,13 @@ module GraphiteAPI
       end
 
       def set time, key, value
-        cache[time.to_i][key] = value.to_f 
-      end
-
-      def incr time, key, value
-        set(time, key, value.to_f + get(time, key))
+        cache[time.to_i][key] = value
       end
 
       private
 
       def cache
-        @cache ||= Hash.new {|h,k| h[k] = Hash.new {|h2,k2| h2[k2] = 0}}
+        @cache ||= Hash.new {|h,k| h[k] = Hash.new}
       end
 
       def clean max_age

data/lib/graphite-api/client.rb

@@ -29,9 +29,9 @@ module GraphiteAPI
       Zscheduler.every( interval ) { block.arity == 1 ? block.call(self) : block.call }
     end
 
-    def metrics metric, time = Time.now
+    def metrics metric, time = nil, aggregation_method = nil
       return if metric.empty?
-      buffer.push :metric => metric, :time => time
+      buffer.push :metric => metric, :time => (time || Time.now), :aggregation_method => aggregation_method
       send_metrics! if options[:direct]
     end
 
@@ -59,7 +59,8 @@ module GraphiteAPI
         :prefix => [],
         :interval => 0,
         :slice => 60,
-        :pid => "/tmp/graphite-middleware.pid"
+        :pid => "/tmp/graphite-middleware.pid",
+        :default_aggregation_method => :sum
       }
     end
 

data/lib/graphite-api/version.rb

@@ -1,3 +1,3 @@
 module GraphiteAPI
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: graphite-api
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Eran Barak Levi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-23 00:00:00.000000000 Z
+date: 2019-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
@@ -78,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: graphite-api
-rubygems_version: 2.6.10
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: Graphite Ruby Client