fluent-plugin-metricsense 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ /vendor/
2
+ /pkg/
3
+ /coverage/
4
+ Gemfile.lock
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source :rubygems
2
+ gemspec
data/README.rdoc ADDED
@@ -0,0 +1,25 @@
1
+ = MetricSense - application metrics aggregation plugin for Fluentd
2
+
3
+ == Example
4
+
5
+ <match metrics.**>
6
+ type metricsense
7
+
8
+ # RDB backend
9
+ type metricsense
10
+ backend rdb_tsdb
11
+ all_segment
12
+ flush_interval 5m
13
+ remove_tag_prefix metrics
14
+ rdb_url mysql2://user:pass@host/mydb
15
+ rdb_table_prefix ms
16
+
17
+ ## Librato Metrics backend
18
+ #all_segment
19
+ #flush_interval 5m
20
+ #remove_tag_prefix metrics
21
+ #backend librato
22
+ #librato_user xxxx@yyyy
23
+ #librato_token XYZXYZ
24
+ </match>
25
+
data/Rakefile ADDED
@@ -0,0 +1,17 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ #require 'rake/testtask'
5
+ #
6
+ #Rake::TestTask.new(:test) do |test|
7
+ # test.libs << 'lib' << 'test'
8
+ # test.test_files = FileList['test/plugin/*.rb']
9
+ # test.verbose = true
10
+ #end
11
+
12
+ #task :coverage do |t|
13
+ # ENV['SIMPLE_COV'] = '1'
14
+ # Rake::Task["test"].invoke
15
+ #end
16
+
17
+ task :default => [:build]
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.0
@@ -0,0 +1,22 @@
1
+ $:.push File.expand_path('../lib', __FILE__)
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = "fluent-plugin-metricsense"
5
+ s.description = "MetricSense - application metrics aggregation plugin for Fluentd"
6
+ s.summary = s.description
7
+ s.homepage = "https://github.com/treasure-data/fluent-plugin-metricsense"
8
+ s.version = File.read("VERSION").strip
9
+ s.authors = ["Sadayuki Furuhashi"]
10
+ s.email = "sf@treasure-data.com"
11
+ s.has_rdoc = false
12
+ s.require_paths = ['lib']
13
+ #s.platform = Gem::Platform::RUBY
14
+ s.files = `git ls-files`.split("\n")
15
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+
18
+ s.add_dependency "fluentd", "~> 0.10.6"
19
+ s.add_development_dependency "rake", ">= 0.8.7"
20
+ s.add_development_dependency 'bundler', ['>= 1.0.0']
21
+ s.add_development_dependency "simplecov", ">= 0.5.4"
22
+ end
@@ -0,0 +1,114 @@
1
+ #
2
+ # fluent-plugin-metricsense
3
+ #
4
+ # Copyright (C) 2012 Sadayuki Furuhashi
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ module Fluent::MetricSenseOutput::Backends
19
+
20
+ require 'net/http'
21
+ require 'cgi'
22
+ require 'json'
23
+
24
+ class LibratoBackend < Fluent::MetricSenseOutput::Backend
25
+ Fluent::MetricSenseOutput::BACKENDS['librato'] = self
26
+
27
+ config_param :librato_user, :string
28
+ config_param :librato_token, :string
29
+
30
+ def initialize
31
+ super
32
+ @initialized_metrics = {}
33
+ end
34
+
35
+ def write(data)
36
+ http = Net::HTTP.new('metrics-api.librato.com', 443)
37
+ http.open_timeout = 60
38
+ http.use_ssl = true
39
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE # FIXME verify
40
+ #http.verify_mode = OpenSSL::SSL::VERIFY_PEER
41
+ http.cert_store = OpenSSL::X509::Store.new
42
+ header = {}
43
+
44
+ begin
45
+ # send upto 50 entries at once
46
+ data.each_slice(50) {|slice|
47
+ req = Net::HTTP::Post.new('/v1/metrics', header)
48
+ req.basic_auth @librato_user, @librato_token
49
+
50
+ data = []
51
+ slice.each_with_index {|(tag,time,value,seg_key,seg_val),i|
52
+ if seg_key
53
+ name = "#{tag}:#{seg_key}"
54
+ source = seg_val
55
+ else
56
+ name = tag
57
+ source = nil
58
+ end
59
+ h = {
60
+ "name" => name,
61
+ "measure_time" => time,
62
+ "value" => value,
63
+ }
64
+ h["source"] = source.to_s if source
65
+ data << h
66
+ ensure_metric_initialized(http, name)
67
+ }
68
+ body = {"gauges"=>data}.to_json
69
+
70
+ $log.trace { "librato metrics: #{data.inspect}" }
71
+ req.body = body
72
+ req.set_content_type("application/json")
73
+ res = http.request(req)
74
+
75
+ # TODO error handling
76
+ if res.code != "200"
77
+ $log.warn "librato_metrics: #{res.code}: #{res.body}"
78
+ end
79
+ }
80
+
81
+ ensure
82
+ http.finish if http.started?
83
+ end
84
+ end
85
+
86
+ METRIC_INITIALIZE_REQUEST = [
87
+ "type=gauge",
88
+ "attributes[source_aggregate]=true",
89
+ "attributes[summarize_function]=sum",
90
+ "attributes[display_stacked]=true",
91
+ ].join('&')
92
+
93
+ def ensure_metric_initialized(http, name)
94
+ return if @initialized_metrics[name]
95
+
96
+ header = {}
97
+ req = Net::HTTP::Put.new("/v1/metrics/#{CGI.escape name}", header)
98
+ req.basic_auth @librato_user, @librato_token
99
+
100
+ $log.trace { "librato initialize metric: #{name}" }
101
+ req.body = METRIC_INITIALIZE_REQUEST
102
+ res = http.request(req)
103
+
104
+ # TODO error handling
105
+ if res.code !~ /20./
106
+ $log.warn "librato_metrics: #{res.code}: #{res.body}"
107
+ else
108
+ @initialized_metrics[name] = true
109
+ end
110
+ end
111
+ end
112
+
113
+ end
114
+
@@ -0,0 +1,307 @@
1
+ #
2
+ # fluent-plugin-metricsense
3
+ #
4
+ # Copyright (C) 2012 Sadayuki Furuhashi
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ module Fluent::MetricSenseOutput::Backends
19
+
20
+ class RDBTSDBBackend < Fluent::MetricSenseOutput::Backend
21
+ include Fluent::Configurable
22
+
23
+ Fluent::MetricSenseOutput::BACKENDS['rdb_tsdb'] = self
24
+
25
+ config_param :rdb_url, :string
26
+ config_param :rdb_table_prefix, :string
27
+ config_param :rdb_read_url, :string, :default => nil
28
+
29
+ INSERT_SUPPRESS_RING_BUFFER_SIZE = 64
30
+
31
+ def initialize
32
+ super
33
+ require 'sequel'
34
+ @insup_ring = []
35
+ @insup_ring_index = 0
36
+ end
37
+
38
+ def configure(conf)
39
+ super
40
+
41
+ @rdb_read_url ||= @rdb_url
42
+
43
+ @metric_tag_table = "#{@rdb_table_prefix}_metric_tags"
44
+ @segment_value_table = "#{@rdb_table_prefix}_segment_values"
45
+ @data_table = "#{@rdb_table_prefix}_data"
46
+ @metric_view = "#{@rdb_table_prefix}_metrics"
47
+ @metric_json_view = "#{@rdb_table_prefix}_json"
48
+
49
+ #sql_standard_concat = lambda {|array| array.join(' || ') }
50
+ #sql_standard_surround = lambda {|expr| "'\"' || #{expr} || '\"'" }
51
+
52
+ case @rdb_url
53
+ when /^mysql/i
54
+ @sql_type = :mysql
55
+ @sql_autoincr_type = "INT"
56
+ @sql_autoincr_ref_type = "INT"
57
+ @sql_autoincr_suffix = " AUTO_INCREMENT"
58
+ @sql_value_type = "SMALLINT"
59
+ @sql_name_type = "VARCHAR(255)"
60
+ @sql_time_type = "INT"
61
+ @sql_insert_ignore = "INSERT IGNORE"
62
+ @sql_insert_returns_last_id = true
63
+ #@sql_concat = lambda {|array| "CONCAT(#{array.join(', ')})" }
64
+ #@sql_surround = lambda {|expr| "CONCAT('\"', #{expr}, '\"')" }
65
+ when /^postgres/i
66
+ @sql_type = :postgresql
67
+ @sql_autoincr_type = "SERIAL"
68
+ @sql_autoincr_ref_type = "INT"
69
+ @sql_autoincr_suffix = ""
70
+ @sql_value_type = "SMALLINT"
71
+ @sql_name_type = "VARCHAR(255)"
72
+ @sql_time_type = "INT"
73
+ @sql_insert_ignore = "INSERT"
74
+ @sql_insert_returns_last_id = false
75
+ #@sql_concat = sql_standard_concat
76
+ #@sql_surround = sql_standard_surround
77
+ when /^sqlite/i
78
+ @sql_type = :sqlite
79
+ @sql_autoincr_type = "INTEGER"
80
+ @sql_autoincr_ref_type = "INTEGER"
81
+ @sql_autoincr_suffix = " AUTOINCREMENT"
82
+ @sql_value_type = "INTEGER"
83
+ @sql_name_type = "TEXT"
84
+ @sql_time_type = "INTEGER"
85
+ @sql_insert_ignore = "INSERT OR IGNORE"
86
+ @sql_insert_returns_last_id = false
87
+ #@sql_concat = sql_standard_concat
88
+ #@sql_surround = sql_standard_surround
89
+ else
90
+ @sql_type = :unknown
91
+ @sql_autoincr_type = "INT"
92
+ @sql_autoincr_ref_type = "INT"
93
+ @sql_autoincr_suffix = " AUTO_INCREMENT"
94
+ @sql_value_type = "SMALLINT"
95
+ @sql_name_type = "VARCHAR(255)"
96
+ @sql_time_type = "INT"
97
+ @sql_insert_ignore = "INSERT IGNORE"
98
+ @sql_insert_returns_last_id = false
99
+ #@sql_concat = sql_standard_concat
100
+ #@sql_surround = sql_standard_surround
101
+ end
102
+ end
103
+
104
+ def start
105
+ ensure_connect do |db|
106
+ db.run %[
107
+ CREATE TABLE IF NOT EXISTS `#{@metric_tag_table}` (
108
+ id #{@sql_autoincr_type} PRIMARY KEY#{@sql_autoincr_suffix},
109
+ metric_name #{@sql_name_type} NOT NULL,
110
+ segment_name #{@sql_name_type} NOT NULL,
111
+ UNIQUE (metric_name, segment_name)
112
+ );]
113
+
114
+ db.run %[
115
+ CREATE TABLE IF NOT EXISTS `#{@segment_value_table}` (
116
+ id #{@sql_autoincr_type} PRIMARY KEY#{@sql_autoincr_suffix},
117
+ name #{@sql_name_type} NOT NULL,
118
+ UNIQUE (name)
119
+ );]
120
+
121
+ minutes = (0..59).to_a.map {|m| "m#{m} #{@sql_value_type} NOT NULL DEFAULT 0" }.join(', ')
122
+ db.run %[
123
+ CREATE TABLE IF NOT EXISTS `#{@data_table}` (
124
+ base_time #{@sql_time_type} NOT NULL,
125
+ metric_id #{@sql_autoincr_ref_type} NOT NULL,
126
+ segment_id #{@sql_autoincr_ref_type},
127
+ #{minutes},
128
+ PRIMARY KEY (base_time, metric_id, segment_id)
129
+ );]
130
+
131
+ if @sql_type == :postgresql
132
+ # ignore duplication error on data_table
133
+ db.run %[
134
+ CREATE OR REPLACE RULE ignore_duplicated_insert AS ON INSERT TO `#{@data_table}`
135
+ WHERE NEW.base_time = OLD.base_time AND NEW.metric_id = OLD.metric_id AND NEW.segment_id = OLD.segment_id
136
+ DO INSTEAD NOTHING;]
137
+ end
138
+
139
+ #minutes = (0..59).to_a.map {|m| "m#{m}" }.join(', ')
140
+ #db.run %[
141
+ # CREATE VIEW IF NOT EXISTS `#{@metric_view}` AS
142
+ # SELECT
143
+ # base_time * 60 AS time,
144
+ # M.metric_name AS metric_name,
145
+ # CASE M.segment_name WHEN '' THEN NULL ELSE M.segment_name END AS segment_name,
146
+ # S.name AS segment_value,
147
+ # #{minutes}
148
+ # FROM `#{@data_table}` T
149
+ # LEFT JOIN `#{@metric_tag_table}` M ON T.metric_id = M.id
150
+ # LEFT JOIN `#{@segment_value_table}` S ON T.segment_id = S.id;]
151
+
152
+ #minutes = (0..59).to_a.map {|m| ["m#{m}", "','"] }.flatten!
153
+ #minutes.pop
154
+ #minutes = @sql_concat.call(["'['"]+minutes+["']'"])
155
+ #db.run %[
156
+ # CREATE VIEW IF NOT EXISTS `#{@metric_json_view}` AS
157
+ # SELECT
158
+ # base_time * 60 AS time,
159
+ # #{@sql_surround.call("M.metric_name")} AS metric_name,
160
+ # CASE WHEN M.segment_name IS NULL OR M.segment_name = '' THEN 'null' ELSE #{@sql_surround.call("M.segment_name")} END AS segment_name,
161
+ # CASE WHEN S.name IS NULL OR S.name = '' THEN 'null' ELSE #{@sql_surround.call("S.name")} END AS segment_value,
162
+ # #{minutes}
163
+ # FROM `#{@data_table}` T
164
+ # LEFT JOIN `#{@metric_tag_table}` M ON T.metric_id = M.id
165
+ # LEFT JOIN `#{@segment_value_table}` S ON T.segment_id = S.id;]
166
+
167
+ reload_metric_names!(db)
168
+ reload_segment_names!(db)
169
+ end
170
+ end
171
+
172
+ #def shutdown
173
+ #end
174
+
175
+ ROW_TIME_WINDOW = 60*24
176
+ ROW_TIME_WINDOW_BITS = 11
177
+ ROW_TIME_WINDOW_MASK = (1<<ROW_TIME_WINDOW_BITS)-1
178
+
179
+ def write(data)
180
+ ensure_connect do |db|
181
+ # group by row_key (base_time,metric_id,segment_id)
182
+ rows = {}
183
+ data.each {|(tag,time,value,seg_key,seg_val)|
184
+ base_time = time / ROW_TIME_WINDOW
185
+ metric_id = get_metric_id(db, tag, seg_key)
186
+ segment_id = get_segment_id(db, seg_val) if seg_val
187
+
188
+ row_key = [base_time, metric_id, segment_id]
189
+ minutes = (rows[row_key] ||= [])
190
+ minutes << ((value << ROW_TIME_WINDOW_BITS) | (time % 60))
191
+ }
192
+
193
+ # insert rows if not exist
194
+ if @sql_type == :sqlite
195
+ # sqlite3 < 1.3.7 doesn't allow multiple rows at once
196
+ rows.keys.each {|row_key|
197
+ next if @insup_ring.include?(row_key)
198
+ db["#{@sql_insert_ignore} INTO `#{@data_table}` (base_time,metric_id,segment_id) VALUES (?,?,?)", *row_key].update
199
+ rid = @insup_ring_index = (@insup_ring_index + 1) % INSERT_SUPPRESS_RING_BUFFER_SIZE
200
+ @insup_ring[rid] = row_key
201
+ }
202
+ else
203
+ insert_sql = "#{@sql_insert_ignore} INTO `#{@data_table}` (base_time,metric_id,segment_id) VALUES " + (["(?,?,?)"] * rows.size).join(', ')
204
+ insert_params = [insert_sql]
205
+ rows.keys.each {|row_key|
206
+ next if @insup_ring.include?(row_key)
207
+ insert_params.concat(row_key)
208
+ rid = @insup_ring_index = (@insup_ring_index + 1) % INSERT_SUPPRESS_RING_BUFFER_SIZE
209
+ @insup_ring[rid] = row_key
210
+ }
211
+ db[*insert_params].update
212
+ end
213
+
214
+ # increment values
215
+ db.transaction do
216
+ rows.each_pair {|row_key,minutes|
217
+ update_sql = "UPDATE `#{@data_table}` SET "
218
+ update_params = [update_sql]
219
+
220
+ values_sql = []
221
+ minutes.each {|m|
222
+ value = m >> ROW_TIME_WINDOW_BITS
223
+ minute = m & ROW_TIME_WINDOW_MASK
224
+ values_sql << "m#{minute} = m#{minute} + ?"
225
+ update_params << value
226
+ }.join(', ')
227
+ update_sql << values_sql.join(', ') << " WHERE base_time=? AND metric_id=? AND segment_id=?"
228
+ update_params.concat(row_key)
229
+
230
+ db[*update_params].update
231
+ }
232
+ end
233
+ end
234
+ end
235
+
236
+ def reload_metric_names!(db)
237
+ map = {}
238
+ db.fetch("SELECT id, metric_name, segment_name FROM `#{@metric_tag_table}`") {|row|
239
+ key = "#{row[:metric_name]}\0#{row[:segment_name]}"
240
+ map[key] = row[:id]
241
+ }
242
+ @metric_names = map
243
+ end
244
+
245
+ def reload_segment_names!(db)
246
+ map = {}
247
+ db.fetch("SELECT id, name FROM `#{@segment_value_table}`") {|row|
248
+ map[row[:name]] = row[:id]
249
+ }
250
+ @segment_names = map
251
+ end
252
+
253
+ def get_metric_id(db, tag, seg_key)
254
+ key = "#{tag}\0#{seg_key}"
255
+ id = @metric_names[key]
256
+ return id if id
257
+
258
+ begin
259
+ id = db["INSERT INTO `#{@metric_tag_table}` (metric_name,segment_name) VALUES (?,?)", tag, seg_key||''].update
260
+ if @sql_insert_returns_last_id
261
+ @metric_names[key] = id
262
+ return id
263
+ end
264
+ reload_metric_names!(db)
265
+ return @metric_names[key]
266
+
267
+ rescue => e
268
+ reload_metric_names!(db)
269
+ id = @metric_names[key]
270
+ return id if id
271
+ raise e
272
+ end
273
+ end
274
+
275
+ def get_segment_id(db, seg_val)
276
+ key = seg_val ? seg_val.to_s : ''
277
+ id = @segment_names[key]
278
+ return id if id
279
+
280
+ begin
281
+ id = db["INSERT INTO `#{@segment_value_table}` (name) VALUES (?)", key].update
282
+ if @sql_insert_returns_last_id
283
+ @segment_names[key] = id
284
+ return id
285
+ end
286
+ reload_segment_names!(db)
287
+ return @segment_names[key]
288
+
289
+ rescue => e
290
+ reload_segment_names!(db)
291
+ id = @segment_names[key]
292
+ return id if id
293
+ raise e
294
+ end
295
+ end
296
+
297
+ def ensure_connect(&block)
298
+ db = Sequel.connect(@rdb_url, :max_connections=>1)
299
+ begin
300
+ block.call(db)
301
+ ensure
302
+ db.disconnect
303
+ end
304
+ end
305
+ end
306
+
307
+ end
@@ -0,0 +1,35 @@
1
+ #
2
+ # fluent-plugin-metricsense
3
+ #
4
+ # Copyright (C) 2012 Sadayuki Furuhashi
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ module Fluent::MetricSenseOutput::Backends
19
+
20
+ class StdoutBackend < Fluent::MetricSenseOutput::Backend
21
+ Fluent::MetricSenseOutput::BACKENDS['stdout'] = self
22
+
23
+ def write(data)
24
+ data.each {|tag,time,value,seg_key,seg_val|
25
+ if seg_key
26
+ puts "#{Time.at(time)} #{tag}: #{value}"
27
+ else
28
+ puts "#{Time.at(time)} #{tag} [#{seg_key}=#{seg_val}]: #{value}"
29
+ end
30
+ }
31
+ end
32
+ end
33
+
34
+ end
35
+
@@ -0,0 +1,162 @@
1
+ # encoding: ascii-8bit
2
+ #
3
+ # fluent-plugin-metricsense
4
+ #
5
+ # Copyright (C) 2012 Sadayuki Furuhashi
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ module Fluent
20
+
21
+ class MetricSenseOutput < Fluent::BufferedOutput
22
+ Fluent::Plugin.register_output('metricsense', self)
23
+
24
+ BACKENDS = {}
25
+
26
+ class Backend
27
+ include Configurable
28
+
29
+ def start
30
+ end
31
+
32
+ def shutdown
33
+ end
34
+ end
35
+
36
+ backend_dir = "#{File.dirname(__FILE__)}/backend"
37
+ Dir.glob("#{backend_dir}/*_backend.rb") {|e|
38
+ require e
39
+ }
40
+
41
+ config_param :segment_keys, :string, :default => nil
42
+ config_param :all_segment, :bool, :default => false
43
+ config_param :value_key, :string, :default => 'value'
44
+
45
+ config_param :backend, :string
46
+
47
+ config_param :remove_tag_prefix, :string, :default => nil
48
+
49
+ def configure(conf)
50
+ super
51
+
52
+ if @remove_tag_prefix
53
+ @remove_tag_prefix = Regexp.new('^' + Regexp.escape(@remove_tag_prefix) + "\\.?")
54
+ end
55
+
56
+ if conf.has_key?('all_segment') && conf['all_segment'].empty?
57
+ @all_segment = true
58
+ end
59
+
60
+ if @all_segment
61
+ @segment_keys = nil
62
+ elsif @segment_keys
63
+ @segment_keys = @segment_keys.strip.split(/\s*,\s*/)
64
+ else
65
+ @segment_keys = []
66
+ end
67
+
68
+ be = BACKENDS[@backend]
69
+ unless be
70
+ raise ConfigError, "unknown backend: #{@backend.inspect}"
71
+ end
72
+
73
+ @backend = be.new
74
+ @backend.configure(conf)
75
+ end
76
+
77
+ def start
78
+ @backend.start
79
+ super
80
+ end
81
+
82
+ def shutdown
83
+ super
84
+ @backend.shutdown
85
+ end
86
+
87
+ def format_stream(tag, es)
88
+ out = ''
89
+ es.each do |time,record|
90
+ value = record[@value_key]
91
+
92
+ fv = value.to_f
93
+ next if fv == 0.0
94
+
95
+ iv = fv.to_i
96
+ if iv.to_f == fv
97
+ value = iv
98
+ else
99
+ value = fv
100
+ end
101
+
102
+ seg_keys = @segment_keys
103
+ unless seg_keys
104
+ seg_keys = record.keys
105
+ seg_keys.delete(@value_key)
106
+ end
107
+
108
+ segs = []
109
+ seg_keys.each {|seg_key|
110
+ if seg_val = record[seg_key]
111
+ segs << seg_key
112
+ segs << seg_val
113
+ end
114
+ }
115
+
116
+ tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
117
+
118
+ [tag, time, value, segs].to_msgpack(out)
119
+ end
120
+ out
121
+ end
122
+
123
+ class SumAggregator
124
+ def initialize
125
+ @value = 0
126
+ end
127
+
128
+ def add(value)
129
+ @value += value
130
+ end
131
+
132
+ attr_reader :value
133
+ end
134
+
135
+ AggregationKey = Struct.new(:tag, :time, :seg_val, :seg_key)
136
+
137
+ def write(chunk)
138
+ counters = {}
139
+
140
+ # select sum(value) from chunk group by tag, time/60, seg_val, seg_key
141
+ chunk.msgpack_each {|tag,time,value,segs|
142
+ time = time / 60 * 60
143
+
144
+ ak = AggregationKey.new(tag, time, nil, nil)
145
+ (counters[ak] ||= SumAggregator.new).add(value)
146
+
147
+ segs.each_slice(2) {|seg_key,seg_val|
148
+ ak = AggregationKey.new(tag, time, seg_val, seg_key)
149
+ (counters[ak] ||= SumAggregator.new).add(value)
150
+ }
151
+ }
152
+
153
+ data = []
154
+ counters.each_pair {|ak,aggr|
155
+ data << [ak.tag, ak.time, aggr.value, ak.seg_key, ak.seg_val]
156
+ }
157
+
158
+ @backend.write(data)
159
+ end
160
+ end
161
+
162
+ end
metadata ADDED
@@ -0,0 +1,119 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-metricsense
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Sadayuki Furuhashi
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-11-08 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: 0.10.6
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: 0.10.6
30
+ - !ruby/object:Gem::Dependency
31
+ name: rake
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: 0.8.7
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: 0.8.7
46
+ - !ruby/object:Gem::Dependency
47
+ name: bundler
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: 1.0.0
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: 1.0.0
62
+ - !ruby/object:Gem::Dependency
63
+ name: simplecov
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: 0.5.4
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: 0.5.4
78
+ description: MetricSense - application metrics aggregation plugin for Fluentd
79
+ email: sf@treasure-data.com
80
+ executables: []
81
+ extensions: []
82
+ extra_rdoc_files: []
83
+ files:
84
+ - .gitignore
85
+ - Gemfile
86
+ - README.rdoc
87
+ - Rakefile
88
+ - VERSION
89
+ - fluent-plugin-metricsense.gemspec
90
+ - lib/fluent/plugin/backend/librato_backend.rb
91
+ - lib/fluent/plugin/backend/rdb_tsdb_backend.rb
92
+ - lib/fluent/plugin/backend/stdout_backend.rb
93
+ - lib/fluent/plugin/out_metricsense.rb
94
+ homepage: https://github.com/treasure-data/fluent-plugin-metricsense
95
+ licenses: []
96
+ post_install_message:
97
+ rdoc_options: []
98
+ require_paths:
99
+ - lib
100
+ required_ruby_version: !ruby/object:Gem::Requirement
101
+ none: false
102
+ requirements:
103
+ - - ! '>='
104
+ - !ruby/object:Gem::Version
105
+ version: '0'
106
+ required_rubygems_version: !ruby/object:Gem::Requirement
107
+ none: false
108
+ requirements:
109
+ - - ! '>='
110
+ - !ruby/object:Gem::Version
111
+ version: '0'
112
+ requirements: []
113
+ rubyforge_project:
114
+ rubygems_version: 1.8.23
115
+ signing_key:
116
+ specification_version: 3
117
+ summary: MetricSense - application metrics aggregation plugin for Fluentd
118
+ test_files: []
119
+ has_rdoc: false