graphite-api 0.3.3 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 8d03b673ba89014ffbe0bb86c4455350c51aaba5
4
- data.tar.gz: 0b257ccfefc1ccb43fd8a1ce76c51e962f55a6a7
2
+ SHA256:
3
+ metadata.gz: 76010192ae0104397b2f2f069f03ee6f5895f42473f8bab79068ce7808104a58
4
+ data.tar.gz: da60c3a2a69ed0d1e35232d4b1e4fb90709a7d16c87d241801824b67094cf4b0
5
5
  SHA512:
6
- metadata.gz: 9441471eb45d0dda1cb34d9c562534295ecb96cc0f2fcd5e8fcabfd75c52666f7818f6aeaae07476f6af2026159174015eb75667e0e4341a4bf98dd9e98f68b9
7
- data.tar.gz: f00e42c5824473da725b9ec89787982f9fe3055bab4512abab23b3bf047501c6c4a21e36ca1231dacec66f4094fdb4a981ba1643566b34d2cf7419b759623b2f
6
+ metadata.gz: 29d966b90c0d676c604e2556a05992667ec6a083e078ae2a2a4cb75a9524ea042919585038ff18dcf1b90b7ed049c414ebefdb5dbc0389787a2976960cf534b9
7
+ data.tar.gz: 4c1c95a572507ecd6df22581559f615fecdefc8cb8c098707bdaac0ed2bc39f51c6dcbeaa13b3dee441a3b36a0e86cbf30d0e916676214aa0aa40aa38480edc8
data/README.md CHANGED
@@ -13,17 +13,9 @@
13
13
  * **Thread-Safe** client.
14
14
 
15
15
  ## Status
16
- <table>
17
- <tr>
18
- <td> Version </td>
19
- <td><a href="https://rubygems.org/gems/graphite-api"><img src=https://fury-badge.herokuapp.com/rb/graphite-api.png></a> </td>
20
- </tr>
21
- <tr>
22
- <td> Build </td>
23
- <td><a href="https://travis-ci.org/kontera-technologies/graphite-api"><img src=https://travis-ci.org/kontera-technologies/graphite-api.png?branch=master></a>
24
- </td>
25
- </tr>
26
- </table>
16
+ [![Gem Version](https://badge.fury.io/rb/graphite-api.svg)](https://badge.fury.io/rb/graphite-api)
17
+ [![Build Status](https://travis-ci.org/kontera-technologies/graphite-api.svg?branch=master)](https://travis-ci.org/kontera-technologies/graphite-api)
18
+ [![Test Coverage](https://codecov.io/gh/kontera-technologies/graphite-api/branch/master/graph/badge.svg)](https://codecov.io/gh/kontera-technologies/graphite-api)
27
19
 
28
20
  ## Installation
29
21
  Install stable version
@@ -32,14 +24,6 @@ Install stable version
32
24
  gem install graphite-api
33
25
  ```
34
26
 
35
- Install the latest from github
36
-
37
- ```
38
- git clone git://github.com/kontera-technologies/graphite-api.git
39
- cd graphite-api
40
- rake install
41
- ```
42
-
43
27
  ## Client Usage
44
28
 
45
29
  Creating a new UDP client
@@ -61,7 +45,11 @@ options = {
61
45
  interval: 60,
62
46
 
63
47
  # Optional: set the max age in seconds for records reanimation ( default is 12 hours )
64
- cache: 4 * 60 * 60
48
+ cache: 4 * 60 * 60,
49
+
50
+ # Optional: The default aggregation method for multiple reports in the same slice (default is :add).
51
+ # Possible options: :sum, :avg, :replace
52
+ default_aggregation_method: :avg
65
53
  }
66
54
 
67
55
  client = GraphiteAPI.new options
@@ -77,6 +65,11 @@ TCP Client with 30 seconds timeout
77
65
  client = GraphiteAPI.new graphite: "tcp://graphite.example.com:2003?timeout=30"
78
66
  ```
79
67
 
68
+ TCP Client with custom aggregation method
69
+ ```ruby
70
+ client = GraphiteAPI.new graphite: "tcp://graphite.example.com:2003", default_aggregation_method: :avg
71
+ ```
72
+
80
73
  Adding simple metrics
81
74
  ```ruby
82
75
  require 'graphite-api'
@@ -108,6 +101,24 @@ client.metrics({
108
101
  # => webServer.web01.memUsage 40 1326067060
109
102
  ```
110
103
 
104
+ Adding metrics with custom aggregation method
105
+ ```ruby
106
+ require 'graphite-api'
107
+
108
+ client = GraphiteAPI.new( graphite: 'udp://graphite:2003' )
109
+
110
+ client.metrics({
111
+ "webServer.web01.loadAvg" => 10,
112
+ "webServer.web01.memUsage" => 40
113
+ },Time.at(1326067060), :avg)
114
+ client.metrics({
115
+ "webServer.web01.loadAvg" => 20,
116
+ "webServer.web01.memUsage" => 50
117
+ },Time.at(1326067060), :avg)
118
+ # => webServer.web01.loadAvg 15 1326067060
119
+ # => webServer.web01.memUsage 45 1326067060
120
+ ```
121
+
111
122
  Verifying connectivity
112
123
  ```ruby
113
124
  require 'graphite-api'
@@ -225,6 +236,7 @@ Usage: graphite-middleware [options]
225
236
  -i, --interval INT report every X seconds (default 60)
226
237
  -s, --slice SECONDS send to graphite in X seconds slices (default 60)
227
238
  -r, --reanimation HOURS reanimate records that are younger than X hours, please see README
239
+ -m, --aggregation-method method The aggregation method (sum, avg or replace) for multiple reports in the same time slice (default sum)
228
240
 
229
241
  More Info @ https://github.com/kontera-technologies/graphite-api
230
242
  ```
data/Rakefile CHANGED
@@ -9,7 +9,7 @@ def message msg
9
9
  puts "*** #{msg} ***"
10
10
  end
11
11
 
12
- task(:test => :functional) { ENV['with_coverage'] = "true" }
12
+ task(:test => :functional)
13
13
 
14
14
  Rake::TestTask.new do |t|
15
15
  t.libs << "tests"
@@ -18,14 +18,14 @@ end
18
18
 
19
19
  task :functional do
20
20
  some_failed = false
21
-
21
+
22
22
  next unless ENV['SKIP_FUNC'].nil?
23
23
 
24
24
  unless RUBY_COPYRIGHT.end_with?("Matsumoto")
25
25
  puts("Functional tests are enabled only on MRI...")
26
26
  next
27
27
  end
28
-
28
+
29
29
  message "Executing GraphiteAPI Functional Tests"
30
30
  message "( You can skip them by passing SKIP_FUNC=true )"
31
31
 
@@ -56,8 +56,3 @@ GraphiteAPI::GemSpec = eval File.read 'graphite-api.gemspec'
56
56
  Gem::PackageTask.new(GraphiteAPI::GemSpec) do |p|
57
57
  p.gem_spec = GraphiteAPI::GemSpec
58
58
  end
59
-
60
- task :install => [:gem] do
61
- sh "gem install pkg/graphite-api"
62
- Rake::Task['clobber_package'].execute
63
- end
@@ -31,7 +31,7 @@ OptionParser.new do |opts|
31
31
  options[:pid] = pid_file
32
32
  end
33
33
 
34
- opts.on("-d", "--daemonize","run in background") do
34
+ opts.on("-d", "--daemonize","run in background") do
35
35
  options[:daemonize] = true
36
36
  end
37
37
 
@@ -47,8 +47,13 @@ OptionParser.new do |opts|
47
47
  (options[:cache] = exp.to_i * 3600) if exp.to_i > 0
48
48
  end
49
49
 
50
+ opts.on("-m", "--aggregation-method method","The aggregation method (sum, avg or replace) for multiple reports in the same time slice (default sum)") do |exp|
51
+ raise "Invalid aggregation method. Valid values are sum, avg or replace." unless ["sum", "avg", "replace"].include? exp
52
+ options[:default_aggregation_method] = exp.to_sym
53
+ end
54
+
50
55
  opts.on("-v", "--version","Show version and exit") do |exp|
51
- puts "Version #{GraphiteAPI.version}"
56
+ puts "Version #{GraphiteAPI.version}"
52
57
  exit
53
58
  end
54
59
 
@@ -3,16 +3,16 @@
3
3
  # Handle Socket & Client data streams
4
4
  # -----------------------------------------------------
5
5
  # Usage:
6
- # buff = GraphiteAPI::Buffer.new(GraphiteAPI::Utils.default_options)
7
- # buff << {:metric => {"load_avg" => 10},:time => Time.now}
8
- # buff << {:metric => {"load_avg" => 30},:time => Time.now}
6
+ # buff = GraphiteAPI::Buffer.new(GraphiteAPI::Client.default_options)
7
+ # buff << {:metric => {"load_avg" => 10},:time => Time.now, :aggregation_method => :avg}
8
+ # buff << {:metric => {"load_avg" => 30},:time => Time.now, :aggregation_method => :avg}
9
9
  # buff.stream "mem.usage 1"
10
10
  # buff.stream "90 1326842563\n"
11
11
  # buff.stream "shuki.tuki 999 1326842563\n"
12
- # buff.pull.each {|o| p o}
12
+ # buff.pull.each {|o| p o}
13
13
  #
14
14
  # Produce:
15
- # ["load_avg", 40.0, 1326881160]
15
+ # ["load_avg", 20.0, 1326881160]
16
16
  # ["mem.usage", 190.0, 1326842520]
17
17
  # ["shuki.tuki", 999.0, 1326842520]
18
18
  # -----------------------------------------------------
@@ -21,38 +21,44 @@ require 'set'
21
21
 
22
22
  module GraphiteAPI
23
23
  class Buffer
24
-
24
+
25
25
  IGNORE = ["\r"]
26
26
  END_OF_STREAM = "\n"
27
27
  VALID_MESSAGE = /^[\w|\.|-]+ \d+(?:\.|\d)* \d+$/
28
-
28
+
29
+ AGGREGATORS = {
30
+ sum: ->(*args) { args.reduce(0) { |sum, x| sum + x } },
31
+ avg: ->(*args) { args.reduce(0) { |sum, x| sum + x } / [args.length, 1].max },
32
+ replace: ->(*args) { args.last },
33
+ }
34
+
29
35
  def initialize options
30
36
  @options = options
31
37
  @queue = Queue.new
32
38
  @streamer = Hash.new {|h,k| h[k] = ""}
33
- @cache = Cache::Memory.new options if options[:cache]
39
+ @cache = Cache::Memory.new(options) if options[:cache]
34
40
  end
35
-
41
+
36
42
  attr_reader :queue, :options, :streamer, :cache
37
-
43
+
38
44
  # this method isn't thread safe
39
45
  # use #push for multiple threads support
40
46
  def stream message, client_id = nil
41
47
  message.gsub(/\t/,' ').each_char do |char|
42
48
  next if invalid_char? char
43
- streamer[client_id] += char
44
-
49
+ streamer[client_id] += char
50
+
45
51
  if closed_stream? streamer[client_id]
46
- if streamer[client_id] =~ VALID_MESSAGE
52
+ if streamer[client_id] =~ VALID_MESSAGE
47
53
  push stream_message_to_obj streamer[client_id]
48
54
  end
49
55
  streamer.delete client_id
50
56
  end
51
57
  end
52
58
  end
53
-
59
+
54
60
  # Add records to buffer
55
- # push({:metric => {'a' => 10},:time => Time.now})
61
+ # push({:metric => {'a' => 10},:time => Time.now,:aggregation_method => :sum})
56
62
  def push obj
57
63
  Logger.debug [:buffer,:add, obj]
58
64
  queue.push obj
@@ -62,49 +68,67 @@ module GraphiteAPI
62
68
  alias_method :<<, :push
63
69
 
64
70
  def pull format = nil
65
- data = Hash.new {|h,k| h[k] = Hash.new {|h2,k2| h2[k2] = 0}}
71
+ data = Hash.new { |h,time| h[time] = Hash.new { |h2,metric| h2[metric] = cache_get(time, metric) } }
72
+ aggregation_methods = Hash.new
66
73
 
67
74
  counter = 0
68
- while new_records?
69
- break if ( counter += 1 ) > 1_000_000 # TODO: fix this
70
- hash = queue.pop
71
- hash[:metric].each {|k,v| data[normalize_time(hash[:time],options[:slice])][k] += v.to_f}
75
+ while new_records? and (counter += 1) < 1_000_000
76
+ metrics, time, method_name = queue.pop.values_at(:metric, :time, :aggregation_method)
77
+
78
+ normalized_time = normalize_time(time, options[:slice])
79
+ metrics.each do |metric, value|
80
+ aggregation_methods[metric] = method_name || options[:default_aggregation_method]
81
+ data[normalized_time][metric].push value.to_f
82
+ cache_set(normalized_time, metric, data[normalized_time][metric])
83
+ end
72
84
  end
73
-
74
- data.map do |time, hash|
75
- hash.map do |key, value|
76
- value = cache.incr(time,key,value) if cache
77
- results = ["#{prefix}#{key}",("%f"%value).to_f, time]
85
+
86
+ data.map do |time, metrics|
87
+ metrics.map do |metric, raw_values|
88
+ value = AGGREGATORS[aggregation_methods[metric]].call(*raw_values)
89
+ results = ["#{prefix}#{metric}",("%f"%value).to_f, time]
78
90
  format == :string ? results.join(" ") : results
79
91
  end
80
92
  end.flatten(1)
81
93
  end
82
-
94
+
83
95
  def inspect
84
- "#<GraphiteAPI::Buffer:%s @quque#size=%s @streamer=%s>" %
96
+ "#<GraphiteAPI::Buffer:%s @quque#size=%s @streamer=%s>" %
85
97
  [ object_id, queue.size, streamer]
86
98
  end
87
-
99
+
88
100
  def new_records?
89
101
  !queue.empty?
90
102
  end
91
103
 
92
104
  private
93
105
 
106
+ def cache_get time, metric
107
+ if cache
108
+ cache.get(time, metric) || []
109
+ else
110
+ []
111
+ end
112
+ end
113
+
114
+ def cache_set time, metric, value
115
+ cache.set(time, metric, value) if cache
116
+ end
117
+
94
118
  def normalize_time time, slice
95
119
  slice = 60 if slice.nil?
96
120
  ((time || Time.now).to_i / slice * slice).to_i
97
121
  end
98
-
122
+
99
123
  def stream_message_to_obj message
100
124
  parts = message.split
101
125
  {:metric => { parts[0] => parts[1] },:time => Time.at(parts[2].to_i) }
102
126
  end
103
-
127
+
104
128
  def invalid_char? char
105
129
  IGNORE.include? char
106
130
  end
107
-
131
+
108
132
  def closed_stream? string
109
133
  string[-1,1] == END_OF_STREAM
110
134
  end
@@ -116,6 +140,6 @@ module GraphiteAPI
116
140
  ""
117
141
  end
118
142
  end
119
-
143
+
120
144
  end
121
145
  end
@@ -11,17 +11,13 @@ module GraphiteAPI
11
11
  end
12
12
 
13
13
  def set time, key, value
14
- cache[time.to_i][key] = value.to_f
15
- end
16
-
17
- def incr time, key, value
18
- set(time, key, value.to_f + get(time, key))
14
+ cache[time.to_i][key] = value
19
15
  end
20
16
 
21
17
  private
22
18
 
23
19
  def cache
24
- @cache ||= Hash.new {|h,k| h[k] = Hash.new {|h2,k2| h2[k2] = 0}}
20
+ @cache ||= Hash.new {|h,k| h[k] = Hash.new}
25
21
  end
26
22
 
27
23
  def clean max_age
@@ -29,9 +29,9 @@ module GraphiteAPI
29
29
  Zscheduler.every( interval ) { block.arity == 1 ? block.call(self) : block.call }
30
30
  end
31
31
 
32
- def metrics metric, time = Time.now
32
+ def metrics metric, time = nil, aggregation_method = nil
33
33
  return if metric.empty?
34
- buffer.push :metric => metric, :time => time
34
+ buffer.push :metric => metric, :time => (time || Time.now), :aggregation_method => aggregation_method
35
35
  send_metrics! if options[:direct]
36
36
  end
37
37
 
@@ -59,7 +59,8 @@ module GraphiteAPI
59
59
  :prefix => [],
60
60
  :interval => 0,
61
61
  :slice => 60,
62
- :pid => "/tmp/graphite-middleware.pid"
62
+ :pid => "/tmp/graphite-middleware.pid",
63
+ :default_aggregation_method => :sum
63
64
  }
64
65
  end
65
66
 
@@ -1,3 +1,3 @@
1
1
  module GraphiteAPI
2
- VERSION = "0.3.3"
2
+ VERSION = "0.4.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: graphite-api
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.3
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eran Barak Levi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-01-23 00:00:00.000000000 Z
11
+ date: 2019-02-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: eventmachine
@@ -78,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
78
78
  version: '0'
79
79
  requirements: []
80
80
  rubyforge_project: graphite-api
81
- rubygems_version: 2.6.10
81
+ rubygems_version: 2.7.8
82
82
  signing_key:
83
83
  specification_version: 4
84
84
  summary: Graphite Ruby Client