fluent-plugin-librato-metrics 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,34 +18,76 @@ Simply use RubyGems:
18
18
  librato_user YOUR_LIBRATO_USER_NAME
19
19
  librato_token YOUR_LIBRATO_TOKEN
20
20
 
21
- # simple key-value metrics
22
- <metrics metrics.status.**>
23
- name status
24
- each_key key
25
- value_key value
26
- </metrics>
27
-
28
- # single-stream event count
21
+ # event count
29
22
  <metrics metrics.event.mytask>
30
23
  name mytask.count
31
24
  </metrics>
32
25
 
33
- # single-stream event sum
26
+ # event sum
34
27
  <metrics metrics.event.mytask>
35
28
  name mytask.size
36
29
  value_key size
37
30
  </metrics>
38
31
 
39
- # single-stream event sum; value type is float
32
+ # event sum with float type
40
33
  <metrics metrics.event.mytask>
41
34
  name mytask.elapsed
42
35
  value_key elapsed
43
36
  type float
44
37
  </metrics>
45
38
 
39
+ # count with source
40
+ <metrics metrics.web_event.4xx>
41
+ name web.status_count
42
+ source 4xx
43
+ </metrics>
44
+ <metrics metrics.web_event.5xx>
45
+ name web.status_count
46
+ source 5xx
47
+ </metrics>
48
+
49
+ # sum with source
50
+ <metrics metrics.web_count>
51
+ name web.status_count
52
+ value_key 4xx_count
53
+ source 4xx
54
+ </metrics>
55
+ <metrics metrics.web_count>
56
+ name web.status_count
57
+ value_key 5xx_count
58
+ source 5xx
59
+ </metrics>
60
+
61
+ # key-value metrics (multi-stream)
62
+ <metrics metrics.status.**>
63
+ name status # actual name is status.$record[key]
64
+ each_key key
65
+ value_key value
66
+ </metrics>
67
+
46
68
  </match>
47
69
 
48
- [librato_user (required)] User name of librato metrics
70
+ === Global configuration
71
+
72
+ [librato_user STRING (required)] User name of librato metrics
73
+
74
+ [librato_token STRING (required)] Token of librato metrics
75
+
76
+ [source STRING] Default source name of metrics
77
+
78
+ === <metrics> configuration
79
+
80
+ [name STRING (required)] Name of this metrics
81
+
82
+ [source STRING] Source name of this metrics
83
+
84
+ [each_key STRING] Append 'record[each_key]' to this metrics name
85
+
86
+ [each_keys COMMA,SEPARATED,STRINGS] Append 'record[key].record[key].record[key]...' to this metrics name
87
+
88
+ [value_key STIRNG] Use 'record[value]' for the value of this metrics. Default is 1.
89
+
90
+ [count_key STIRNG] Use 'record[value] * record[count_key]' for the value of this metrics. Default is 1.
49
91
 
50
- [librato_token (required)] Token of librato metrics
92
+ [type STIRNG] 'int' or 'float'. Default is int.
51
93
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.2
1
+ 0.2.3
@@ -37,6 +37,17 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
37
37
  super
38
38
  end
39
39
 
40
+ class LibratoMetricsMetrics < Metrics
41
+ config_param :source, :string, :default => nil
42
+ end
43
+
44
+ # override
45
+ def new_metrics(conf, pattern)
46
+ m = LibratoMetricsMetrics.new(pattern)
47
+ m.configure(conf)
48
+ m
49
+ end
50
+
40
51
  def format(tag, time, record)
41
52
  out = ''.force_encoding('ASCII-8BIT')
42
53
  each_metrics(tag, time, record) {|d|
@@ -47,26 +58,35 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
47
58
  key = "#{name}.#{d.keys.join('.')}"
48
59
  end
49
60
  partitioned_time = time / 60 * 60
50
- [key, partitioned_time, d.value*d.count].to_msgpack(out)
61
+ value = d.value * d.count
62
+ source = d.metrics.source
63
+ [key, partitioned_time, value, source].to_msgpack(out)
51
64
  }
52
65
  out
53
66
  end
54
67
 
68
+ AggregationKey = Struct.new(:name, :time, :source)
69
+
55
70
  def write(chunk)
56
- counters = {} #=> {partitioned_time => {name => CounterAggregator}}
57
- gauges = {} #=> {partitioned_time => {name => GaugeAggregator}}
58
-
59
- chunk.msgpack_each {|key,partitioned_time,value|
60
- if m = /^counter\.(.*)$/.match(key)
61
- name = m[1]
62
- ((counters[partitioned_time] ||= {})[name] ||= CounterAggregator.new).add(value)
63
- elsif m = /^gauge\.(.*)$/.match(key)
64
- name = m[1]
65
- ((gauges[partitioned_time] ||= {})[name] ||= GaugeAggregator.new).add(value)
66
- else
71
+ counters = {} #=> {AggregationKey => XxxAggregator}
72
+ gauges = {} #=> {AggregationKey => XxxAggregator}
73
+
74
+ chunk.msgpack_each {|key,time,value,source|
75
+ #if m = /^counter\.(.*)$/.match(key)
76
+ # # FIXME what's the actual behavior of counters?
77
+ # name = m[1]
78
+ # k = AggregationKey.new(name, time, source)
79
+ # (counters[k] ||= MaxAggregator.new).add(value)
80
+ #elsif m = /^gauge\.(.*)$/.match(key)
81
+ # # FIXME what's the actual behavior of gauges?
82
+ # name = m[1]
83
+ # k = AggregationKey.new(name, time, source)
84
+ # (gauges[k] ||= AverageAggregator.new).add(value)
85
+ #else
67
86
  name = key
68
- ((gauges[partitioned_time] ||= {})[name] ||= GaugeAggregator.new).add(value)
69
- end
87
+ k = AggregationKey.new(name, time, source)
88
+ (gauges[k] ||= SumAggregator.new).add(value)
89
+ #end
70
90
  }
71
91
 
72
92
  #http = Net::HTTP.new('metrics-api.librato.com', 80)
@@ -78,18 +98,19 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
78
98
  header = {}
79
99
 
80
100
  begin
81
- {'counters'=>counters, 'gauges'=>gauges}.each_pair {|type,partitions|
82
- partitions.each_pair {|partitioned_time,name_aggrs|
101
+ {'counters'=>counters, 'gauges'=>gauges}.each_pair {|type,k_aggrs|
102
+ # send upto 10 entries at once
103
+ k_aggrs.each_slice(10) {|k_aggrs_slice|
83
104
  req = Net::HTTP::Post.new('/v1/metrics', header)
84
105
  req.basic_auth @librato_user, @librato_token
85
106
 
86
- params = {
87
- 'measure_time' => partitioned_time.to_s,
88
- }
89
- params['source'] = @source if @source
107
+ params = {}
108
+ params['source'] = @source if @source # default source
90
109
 
91
- name_aggrs.each_with_index {|(name,aggr),i|
92
- params["#{type}[#{i}][name]"] = name
110
+ k_aggrs_slice.each_with_index {|(k,aggr),i|
111
+ params["#{type}[#{i}][name]"] = k.name
112
+ params["#{type}[#{i}][measure_time]"] = k.time.to_s
113
+ params["#{type}[#{i}][source]"] = k.source if k.source
93
114
  params["#{type}[#{i}][value]"] = aggr.get.to_s
94
115
  }
95
116
 
@@ -110,7 +131,7 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
110
131
 
111
132
 
112
133
  # max(value) aggregator
113
- class CounterAggregator
134
+ class MaxAggregator
114
135
  def initialize
115
136
  @value = 0
116
137
  end
@@ -126,8 +147,23 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
126
147
  end
127
148
  end
128
149
 
150
+ # sum(value) aggregator
151
+ class SumAggregator
152
+ def initialize
153
+ @value = 0
154
+ end
155
+
156
+ def add(value)
157
+ @value += value
158
+ end
159
+
160
+ def get
161
+ @value
162
+ end
163
+ end
164
+
129
165
  # avg(value) aggregator
130
- class GaugeAggregator
166
+ class AverageAggregator
131
167
  def initialize
132
168
  @value = 0
133
169
  @num = 0
@@ -46,7 +46,7 @@ module LibratoMetricsOutput::MetricsMixin
46
46
  e.name == 'metrics'
47
47
  }.each {|e|
48
48
  defaults.each_pair {|k,v| e[k] = v if v != nil }
49
- add_metrics(e, e.arg)
49
+ @metrics << new_metrics(e, e.arg)
50
50
  }
51
51
 
52
52
  if mk = @match_key
@@ -56,10 +56,9 @@ module LibratoMetricsOutput::MetricsMixin
56
56
  end
57
57
  end
58
58
 
59
- def add_metrics(conf, pattern)
59
+ def new_metrics(conf, pattern)
60
60
  m = Metrics.new(pattern)
61
61
  m.configure(conf)
62
- @metrics << m
63
62
  end
64
63
 
65
64
  class Data
@@ -185,18 +184,25 @@ module LibratoMetricsOutput::MetricsMixin
185
184
  def evaluate(tag, time, record)
186
185
  return nil unless @match_proc.call(tag)
187
186
 
188
- data = @data
189
-
190
- keys = data.keys = @key_proc.call(record)
187
+ keys = @key_proc.call(record)
191
188
  return nil unless keys
192
189
 
193
- value = data.value = @value_proc.call(record)
190
+ value = @value_proc.call(record)
194
191
  return nil unless value
195
192
 
196
- data.count = @count_proc.call(record)
193
+ count = @count_proc.call(record)
194
+
195
+ data = new_data
196
+ data.keys = keys
197
+ data.value = value
198
+ data.count = count
197
199
 
198
200
  return data
199
201
  end
202
+
203
+ def new_data
204
+ Data.new(self)
205
+ end
200
206
  end
201
207
 
202
208
  def each_metrics(tag, time, record, &block)
@@ -36,6 +36,13 @@ class LibratoMetricsOutputTest < Test::Unit::TestCase
36
36
  value_key elapsed
37
37
  type float
38
38
  </metrics>
39
+
40
+ # single-stream event sum with source
41
+ <metrics test5.**>
42
+ name test5.stack
43
+ value_key count
44
+ source type1
45
+ </metrics>
39
46
  ]
40
47
 
41
48
  TIME = Time.parse("2011-01-02 13:14:15 UTC").to_i
@@ -57,23 +64,28 @@ class LibratoMetricsOutputTest < Test::Unit::TestCase
57
64
  def test_format
58
65
  d = create_driver("test1.1")
59
66
  d.emit({"key"=>"a", "value"=>10, "count"=>2}, TIME)
60
- d.expect_format ["test1.a", TIME/60*60, 20].to_msgpack
67
+ d.expect_format ["test1.a", TIME/60*60, 20, nil].to_msgpack
61
68
  d.run
62
69
 
63
70
  d = create_driver("test2.2")
64
71
  d.emit({}, TIME)
65
- d.expect_format ["test2.count", TIME/60*60, 1].to_msgpack
72
+ d.expect_format ["test2.count", TIME/60*60, 1, nil].to_msgpack
66
73
  d.run
67
74
 
68
75
  d = create_driver("test3.3")
69
76
  d.emit({"size"=>2100}, TIME)
70
- d.expect_format ["test3.size", TIME/60*60, 2100].to_msgpack
77
+ d.expect_format ["test3.size", TIME/60*60, 2100, nil].to_msgpack
71
78
  d.run
72
79
 
73
80
  d = create_driver("test3.test4")
74
81
  d.emit({"size"=>2400, "elapsed"=>10.0}, TIME)
75
- d.expect_format ["test3.size", TIME/60*60, 2400].to_msgpack
76
- d.expect_format ["test4.elapsed", TIME/60*60, 10.0].to_msgpack
82
+ d.expect_format ["test3.size", TIME/60*60, 2400, nil].to_msgpack
83
+ d.expect_format ["test4.elapsed", TIME/60*60, 10.0, nil].to_msgpack
84
+ d.run
85
+
86
+ d = create_driver("test5")
87
+ d.emit({"count"=>12}, TIME)
88
+ d.expect_format ["test5.stack", TIME/60*60, 12, "type1"].to_msgpack
77
89
  d.run
78
90
  end
79
91
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-librato-metrics
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -13,7 +13,7 @@ date: 2012-03-30 00:00:00.000000000Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
16
- requirement: &70131020756540 !ruby/object:Gem::Requirement
16
+ requirement: &70308779156220 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ~>
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: 0.10.0
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70131020756540
24
+ version_requirements: *70308779156220
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: rake
27
- requirement: &70131020756060 !ruby/object:Gem::Requirement
27
+ requirement: &70308779155740 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
32
32
  version: 0.9.2
33
33
  type: :development
34
34
  prerelease: false
35
- version_requirements: *70131020756060
35
+ version_requirements: *70308779155740
36
36
  description: Librato metrics output plugin for Fluent event collector
37
37
  email:
38
38
  - frsyuki@gmail.com
@@ -64,7 +64,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
64
64
  version: '0'
65
65
  segments:
66
66
  - 0
67
- hash: -197996435032986197
67
+ hash: 3018102368853295399
68
68
  required_rubygems_version: !ruby/object:Gem::Requirement
69
69
  none: false
70
70
  requirements:
@@ -73,7 +73,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
73
73
  version: '0'
74
74
  segments:
75
75
  - 0
76
- hash: -197996435032986197
76
+ hash: 3018102368853295399
77
77
  requirements: []
78
78
  rubyforge_project:
79
79
  rubygems_version: 1.8.12