fluent-plugin-metricsense 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ /vendor/
2
+ /pkg/
3
+ /coverage/
4
+ Gemfile.lock
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source :rubygems
2
+ gemspec
data/README.rdoc ADDED
@@ -0,0 +1,25 @@
1
+ = MetricSense - application metrics aggregation plugin for Fluentd
2
+
3
+ == Example
4
+
5
+ <match metrics.**>
6
+ type metricsense
7
+
8
+ # RDB backend
9
+ type metricsense
10
+ backend rdb_tsdb
11
+ all_segment
12
+ flush_interval 5m
13
+ remove_tag_prefix metrics
14
+ rdb_url mysql2://user:pass@host/mydb
15
+ rdb_table_prefix ms
16
+
17
+ ## Librato Metrics backend
18
+ #all_segment
19
+ #flush_interval 5m
20
+ #remove_tag_prefix metrics
21
+ #backend librato
22
+ #librato_user xxxx@yyyy
23
+ #librato_token XYZXYZ
24
+ </match>
25
+
data/Rakefile ADDED
@@ -0,0 +1,17 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ #require 'rake/testtask'
5
+ #
6
+ #Rake::TestTask.new(:test) do |test|
7
+ # test.libs << 'lib' << 'test'
8
+ # test.test_files = FileList['test/plugin/*.rb']
9
+ # test.verbose = true
10
+ #end
11
+
12
+ #task :coverage do |t|
13
+ # ENV['SIMPLE_COV'] = '1'
14
+ # Rake::Task["test"].invoke
15
+ #end
16
+
17
+ task :default => [:build]
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.0
@@ -0,0 +1,22 @@
1
+ $:.push File.expand_path('../lib', __FILE__)
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = "fluent-plugin-metricsense"
5
+ s.description = "MetricSense - application metrics aggregation plugin for Fluentd"
6
+ s.summary = s.description
7
+ s.homepage = "https://github.com/treasure-data/fluent-plugin-metricsense"
8
+ s.version = File.read("VERSION").strip
9
+ s.authors = ["Sadayuki Furuhashi"]
10
+ s.email = "sf@treasure-data.com"
11
+ s.has_rdoc = false
12
+ s.require_paths = ['lib']
13
+ #s.platform = Gem::Platform::RUBY
14
+ s.files = `git ls-files`.split("\n")
15
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+
18
+ s.add_dependency "fluentd", "~> 0.10.6"
19
+ s.add_development_dependency "rake", ">= 0.8.7"
20
+ s.add_development_dependency 'bundler', ['>= 1.0.0']
21
+ s.add_development_dependency "simplecov", ">= 0.5.4"
22
+ end
@@ -0,0 +1,114 @@
1
+ #
2
+ # fluent-plugin-metricsense
3
+ #
4
+ # Copyright (C) 2012 Sadayuki Furuhashi
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ module Fluent::MetricSenseOutput::Backends
19
+
20
+ require 'net/http'
21
+ require 'cgi'
22
+ require 'json'
23
+
24
+ class LibratoBackend < Fluent::MetricSenseOutput::Backend
25
+ Fluent::MetricSenseOutput::BACKENDS['librato'] = self
26
+
27
+ config_param :librato_user, :string
28
+ config_param :librato_token, :string
29
+
30
+ def initialize
31
+ super
32
+ @initialized_metrics = {}
33
+ end
34
+
35
+ def write(data)
36
+ http = Net::HTTP.new('metrics-api.librato.com', 443)
37
+ http.open_timeout = 60
38
+ http.use_ssl = true
39
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE # FIXME verify
40
+ #http.verify_mode = OpenSSL::SSL::VERIFY_PEER
41
+ http.cert_store = OpenSSL::X509::Store.new
42
+ header = {}
43
+
44
+ begin
45
+ # send upto 50 entries at once
46
+ data.each_slice(50) {|slice|
47
+ req = Net::HTTP::Post.new('/v1/metrics', header)
48
+ req.basic_auth @librato_user, @librato_token
49
+
50
+ data = []
51
+ slice.each_with_index {|(tag,time,value,seg_key,seg_val),i|
52
+ if seg_key
53
+ name = "#{tag}:#{seg_key}"
54
+ source = seg_val
55
+ else
56
+ name = tag
57
+ source = nil
58
+ end
59
+ h = {
60
+ "name" => name,
61
+ "measure_time" => time,
62
+ "value" => value,
63
+ }
64
+ h["source"] = source.to_s if source
65
+ data << h
66
+ ensure_metric_initialized(http, name)
67
+ }
68
+ body = {"gauges"=>data}.to_json
69
+
70
+ $log.trace { "librato metrics: #{data.inspect}" }
71
+ req.body = body
72
+ req.set_content_type("application/json")
73
+ res = http.request(req)
74
+
75
+ # TODO error handling
76
+ if res.code != "200"
77
+ $log.warn "librato_metrics: #{res.code}: #{res.body}"
78
+ end
79
+ }
80
+
81
+ ensure
82
+ http.finish if http.started?
83
+ end
84
+ end
85
+
86
+ METRIC_INITIALIZE_REQUEST = [
87
+ "type=gauge",
88
+ "attributes[source_aggregate]=true",
89
+ "attributes[summarize_function]=sum",
90
+ "attributes[display_stacked]=true",
91
+ ].join('&')
92
+
93
+ def ensure_metric_initialized(http, name)
94
+ return if @initialized_metrics[name]
95
+
96
+ header = {}
97
+ req = Net::HTTP::Put.new("/v1/metrics/#{CGI.escape name}", header)
98
+ req.basic_auth @librato_user, @librato_token
99
+
100
+ $log.trace { "librato initialize metric: #{name}" }
101
+ req.body = METRIC_INITIALIZE_REQUEST
102
+ res = http.request(req)
103
+
104
+ # TODO error handling
105
+ if res.code !~ /20./
106
+ $log.warn "librato_metrics: #{res.code}: #{res.body}"
107
+ else
108
+ @initialized_metrics[name] = true
109
+ end
110
+ end
111
+ end
112
+
113
+ end
114
+
@@ -0,0 +1,307 @@
1
+ #
2
+ # fluent-plugin-metricsense
3
+ #
4
+ # Copyright (C) 2012 Sadayuki Furuhashi
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ module Fluent::MetricSenseOutput::Backends
19
+
20
+ class RDBTSDBBackend < Fluent::MetricSenseOutput::Backend
21
+ include Fluent::Configurable
22
+
23
+ Fluent::MetricSenseOutput::BACKENDS['rdb_tsdb'] = self
24
+
25
+ config_param :rdb_url, :string
26
+ config_param :rdb_table_prefix, :string
27
+ config_param :rdb_read_url, :string, :default => nil
28
+
29
+ INSERT_SUPPRESS_RING_BUFFER_SIZE = 64
30
+
31
+ def initialize
32
+ super
33
+ require 'sequel'
34
+ @insup_ring = []
35
+ @insup_ring_index = 0
36
+ end
37
+
38
+ def configure(conf)
39
+ super
40
+
41
+ @rdb_read_url ||= @rdb_url
42
+
43
+ @metric_tag_table = "#{@rdb_table_prefix}_metric_tags"
44
+ @segment_value_table = "#{@rdb_table_prefix}_segment_values"
45
+ @data_table = "#{@rdb_table_prefix}_data"
46
+ @metric_view = "#{@rdb_table_prefix}_metrics"
47
+ @metric_json_view = "#{@rdb_table_prefix}_json"
48
+
49
+ #sql_standard_concat = lambda {|array| array.join(' || ') }
50
+ #sql_standard_surround = lambda {|expr| "'\"' || #{expr} || '\"'" }
51
+
52
+ case @rdb_url
53
+ when /^mysql/i
54
+ @sql_type = :mysql
55
+ @sql_autoincr_type = "INT"
56
+ @sql_autoincr_ref_type = "INT"
57
+ @sql_autoincr_suffix = " AUTO_INCREMENT"
58
+ @sql_value_type = "SMALLINT"
59
+ @sql_name_type = "VARCHAR(255)"
60
+ @sql_time_type = "INT"
61
+ @sql_insert_ignore = "INSERT IGNORE"
62
+ @sql_insert_returns_last_id = true
63
+ #@sql_concat = lambda {|array| "CONCAT(#{array.join(', ')})" }
64
+ #@sql_surround = lambda {|expr| "CONCAT('\"', #{expr}, '\"')" }
65
+ when /^postgres/i
66
+ @sql_type = :postgresql
67
+ @sql_autoincr_type = "SERIAL"
68
+ @sql_autoincr_ref_type = "INT"
69
+ @sql_autoincr_suffix = ""
70
+ @sql_value_type = "SMALLINT"
71
+ @sql_name_type = "VARCHAR(255)"
72
+ @sql_time_type = "INT"
73
+ @sql_insert_ignore = "INSERT"
74
+ @sql_insert_returns_last_id = false
75
+ #@sql_concat = sql_standard_concat
76
+ #@sql_surround = sql_standard_surround
77
+ when /^sqlite/i
78
+ @sql_type = :sqlite
79
+ @sql_autoincr_type = "INTEGER"
80
+ @sql_autoincr_ref_type = "INTEGER"
81
+ @sql_autoincr_suffix = " AUTOINCREMENT"
82
+ @sql_value_type = "INTEGER"
83
+ @sql_name_type = "TEXT"
84
+ @sql_time_type = "INTEGER"
85
+ @sql_insert_ignore = "INSERT OR IGNORE"
86
+ @sql_insert_returns_last_id = false
87
+ #@sql_concat = sql_standard_concat
88
+ #@sql_surround = sql_standard_surround
89
+ else
90
+ @sql_type = :unknown
91
+ @sql_autoincr_type = "INT"
92
+ @sql_autoincr_ref_type = "INT"
93
+ @sql_autoincr_suffix = " AUTO_INCREMENT"
94
+ @sql_value_type = "SMALLINT"
95
+ @sql_name_type = "VARCHAR(255)"
96
+ @sql_time_type = "INT"
97
+ @sql_insert_ignore = "INSERT IGNORE"
98
+ @sql_insert_returns_last_id = false
99
+ #@sql_concat = sql_standard_concat
100
+ #@sql_surround = sql_standard_surround
101
+ end
102
+ end
103
+
104
+ def start
105
+ ensure_connect do |db|
106
+ db.run %[
107
+ CREATE TABLE IF NOT EXISTS `#{@metric_tag_table}` (
108
+ id #{@sql_autoincr_type} PRIMARY KEY#{@sql_autoincr_suffix},
109
+ metric_name #{@sql_name_type} NOT NULL,
110
+ segment_name #{@sql_name_type} NOT NULL,
111
+ UNIQUE (metric_name, segment_name)
112
+ );]
113
+
114
+ db.run %[
115
+ CREATE TABLE IF NOT EXISTS `#{@segment_value_table}` (
116
+ id #{@sql_autoincr_type} PRIMARY KEY#{@sql_autoincr_suffix},
117
+ name #{@sql_name_type} NOT NULL,
118
+ UNIQUE (name)
119
+ );]
120
+
121
+ minutes = (0..59).to_a.map {|m| "m#{m} #{@sql_value_type} NOT NULL DEFAULT 0" }.join(', ')
122
+ db.run %[
123
+ CREATE TABLE IF NOT EXISTS `#{@data_table}` (
124
+ base_time #{@sql_time_type} NOT NULL,
125
+ metric_id #{@sql_autoincr_ref_type} NOT NULL,
126
+ segment_id #{@sql_autoincr_ref_type},
127
+ #{minutes},
128
+ PRIMARY KEY (base_time, metric_id, segment_id)
129
+ );]
130
+
131
+ if @sql_type == :postgresql
132
+ # ignore duplication error on data_table
133
+ db.run %[
134
+ CREATE OR REPLACE RULE ignore_duplicated_insert AS ON INSERT TO `#{@data_table}`
135
+ WHERE NEW.base_time = OLD.base_time AND NEW.metric_id = OLD.metric_id AND NEW.segment_id = OLD.segment_id
136
+ DO INSTEAD NOTHING;]
137
+ end
138
+
139
+ #minutes = (0..59).to_a.map {|m| "m#{m}" }.join(', ')
140
+ #db.run %[
141
+ # CREATE VIEW IF NOT EXISTS `#{@metric_view}` AS
142
+ # SELECT
143
+ # base_time * 60 AS time,
144
+ # M.metric_name AS metric_name,
145
+ # CASE M.segment_name WHEN '' THEN NULL ELSE M.segment_name END AS segment_name,
146
+ # S.name AS segment_value,
147
+ # #{minutes}
148
+ # FROM `#{@data_table}` T
149
+ # LEFT JOIN `#{@metric_tag_table}` M ON T.metric_id = M.id
150
+ # LEFT JOIN `#{@segment_value_table}` S ON T.segment_id = S.id;]
151
+
152
+ #minutes = (0..59).to_a.map {|m| ["m#{m}", "','"] }.flatten!
153
+ #minutes.pop
154
+ #minutes = @sql_concat.call(["'['"]+minutes+["']'"])
155
+ #db.run %[
156
+ # CREATE VIEW IF NOT EXISTS `#{@metric_json_view}` AS
157
+ # SELECT
158
+ # base_time * 60 AS time,
159
+ # #{@sql_surround.call("M.metric_name")} AS metric_name,
160
+ # CASE WHEN M.segment_name IS NULL OR M.segment_name = '' THEN 'null' ELSE #{@sql_surround.call("M.segment_name")} END AS segment_name,
161
+ # CASE WHEN S.name IS NULL OR S.name = '' THEN 'null' ELSE #{@sql_surround.call("S.name")} END AS segment_value,
162
+ # #{minutes}
163
+ # FROM `#{@data_table}` T
164
+ # LEFT JOIN `#{@metric_tag_table}` M ON T.metric_id = M.id
165
+ # LEFT JOIN `#{@segment_value_table}` S ON T.segment_id = S.id;]
166
+
167
+ reload_metric_names!(db)
168
+ reload_segment_names!(db)
169
+ end
170
+ end
171
+
172
+ #def shutdown
173
+ #end
174
+
175
+ ROW_TIME_WINDOW = 60*24
176
+ ROW_TIME_WINDOW_BITS = 11
177
+ ROW_TIME_WINDOW_MASK = (1<<ROW_TIME_WINDOW_BITS)-1
178
+
179
+ def write(data)
180
+ ensure_connect do |db|
181
+ # group by row_key (base_time,metric_id,segment_id)
182
+ rows = {}
183
+ data.each {|(tag,time,value,seg_key,seg_val)|
184
+ base_time = time / ROW_TIME_WINDOW
185
+ metric_id = get_metric_id(db, tag, seg_key)
186
+ segment_id = get_segment_id(db, seg_val) if seg_val
187
+
188
+ row_key = [base_time, metric_id, segment_id]
189
+ minutes = (rows[row_key] ||= [])
190
+ minutes << ((value << ROW_TIME_WINDOW_BITS) | (time % 60))
191
+ }
192
+
193
+ # insert rows if not exist
194
+ if @sql_type == :sqlite
195
+ # sqlite3 < 1.3.7 doesn't allow multiple rows at once
196
+ rows.keys.each {|row_key|
197
+ next if @insup_ring.include?(row_key)
198
+ db["#{@sql_insert_ignore} INTO `#{@data_table}` (base_time,metric_id,segment_id) VALUES (?,?,?)", *row_key].update
199
+ rid = @insup_ring_index = (@insup_ring_index + 1) % INSERT_SUPPRESS_RING_BUFFER_SIZE
200
+ @insup_ring[rid] = row_key
201
+ }
202
+ else
203
+ insert_sql = "#{@sql_insert_ignore} INTO `#{@data_table}` (base_time,metric_id,segment_id) VALUES " + (["(?,?,?)"] * rows.size).join(', ')
204
+ insert_params = [insert_sql]
205
+ rows.keys.each {|row_key|
206
+ next if @insup_ring.include?(row_key)
207
+ insert_params.concat(row_key)
208
+ rid = @insup_ring_index = (@insup_ring_index + 1) % INSERT_SUPPRESS_RING_BUFFER_SIZE
209
+ @insup_ring[rid] = row_key
210
+ }
211
+ db[*insert_params].update
212
+ end
213
+
214
+ # increment values
215
+ db.transaction do
216
+ rows.each_pair {|row_key,minutes|
217
+ update_sql = "UPDATE `#{@data_table}` SET "
218
+ update_params = [update_sql]
219
+
220
+ values_sql = []
221
+ minutes.each {|m|
222
+ value = m >> ROW_TIME_WINDOW_BITS
223
+ minute = m & ROW_TIME_WINDOW_MASK
224
+ values_sql << "m#{minute} = m#{minute} + ?"
225
+ update_params << value
226
+ }.join(', ')
227
+ update_sql << values_sql.join(', ') << " WHERE base_time=? AND metric_id=? AND segment_id=?"
228
+ update_params.concat(row_key)
229
+
230
+ db[*update_params].update
231
+ }
232
+ end
233
+ end
234
+ end
235
+
236
+ def reload_metric_names!(db)
237
+ map = {}
238
+ db.fetch("SELECT id, metric_name, segment_name FROM `#{@metric_tag_table}`") {|row|
239
+ key = "#{row[:metric_name]}\0#{row[:segment_name]}"
240
+ map[key] = row[:id]
241
+ }
242
+ @metric_names = map
243
+ end
244
+
245
+ def reload_segment_names!(db)
246
+ map = {}
247
+ db.fetch("SELECT id, name FROM `#{@segment_value_table}`") {|row|
248
+ map[row[:name]] = row[:id]
249
+ }
250
+ @segment_names = map
251
+ end
252
+
253
+ def get_metric_id(db, tag, seg_key)
254
+ key = "#{tag}\0#{seg_key}"
255
+ id = @metric_names[key]
256
+ return id if id
257
+
258
+ begin
259
+ id = db["INSERT INTO `#{@metric_tag_table}` (metric_name,segment_name) VALUES (?,?)", tag, seg_key||''].update
260
+ if @sql_insert_returns_last_id
261
+ @metric_names[key] = id
262
+ return id
263
+ end
264
+ reload_metric_names!(db)
265
+ return @metric_names[key]
266
+
267
+ rescue => e
268
+ reload_metric_names!(db)
269
+ id = @metric_names[key]
270
+ return id if id
271
+ raise e
272
+ end
273
+ end
274
+
275
+ def get_segment_id(db, seg_val)
276
+ key = seg_val ? seg_val.to_s : ''
277
+ id = @segment_names[key]
278
+ return id if id
279
+
280
+ begin
281
+ id = db["INSERT INTO `#{@segment_value_table}` (name) VALUES (?)", key].update
282
+ if @sql_insert_returns_last_id
283
+ @segment_names[key] = id
284
+ return id
285
+ end
286
+ reload_segment_names!(db)
287
+ return @segment_names[key]
288
+
289
+ rescue => e
290
+ reload_segment_names!(db)
291
+ id = @segment_names[key]
292
+ return id if id
293
+ raise e
294
+ end
295
+ end
296
+
297
+ def ensure_connect(&block)
298
+ db = Sequel.connect(@rdb_url, :max_connections=>1)
299
+ begin
300
+ block.call(db)
301
+ ensure
302
+ db.disconnect
303
+ end
304
+ end
305
+ end
306
+
307
+ end
@@ -0,0 +1,35 @@
1
+ #
2
+ # fluent-plugin-metricsense
3
+ #
4
+ # Copyright (C) 2012 Sadayuki Furuhashi
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ module Fluent::MetricSenseOutput::Backends
19
+
20
+ class StdoutBackend < Fluent::MetricSenseOutput::Backend
21
+ Fluent::MetricSenseOutput::BACKENDS['stdout'] = self
22
+
23
+ def write(data)
24
+ data.each {|tag,time,value,seg_key,seg_val|
25
+ if seg_key
26
+ puts "#{Time.at(time)} #{tag}: #{value}"
27
+ else
28
+ puts "#{Time.at(time)} #{tag} [#{seg_key}=#{seg_val}]: #{value}"
29
+ end
30
+ }
31
+ end
32
+ end
33
+
34
+ end
35
+
@@ -0,0 +1,162 @@
1
+ # encoding: ascii-8bit
2
+ #
3
+ # fluent-plugin-metricsense
4
+ #
5
+ # Copyright (C) 2012 Sadayuki Furuhashi
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ module Fluent
20
+
21
+ class MetricSenseOutput < Fluent::BufferedOutput
22
+ Fluent::Plugin.register_output('metricsense', self)
23
+
24
+ BACKENDS = {}
25
+
26
+ class Backend
27
+ include Configurable
28
+
29
+ def start
30
+ end
31
+
32
+ def shutdown
33
+ end
34
+ end
35
+
36
+ backend_dir = "#{File.dirname(__FILE__)}/backend"
37
+ Dir.glob("#{backend_dir}/*_backend.rb") {|e|
38
+ require e
39
+ }
40
+
41
+ config_param :segment_keys, :string, :default => nil
42
+ config_param :all_segment, :bool, :default => false
43
+ config_param :value_key, :string, :default => 'value'
44
+
45
+ config_param :backend, :string
46
+
47
+ config_param :remove_tag_prefix, :string, :default => nil
48
+
49
+ def configure(conf)
50
+ super
51
+
52
+ if @remove_tag_prefix
53
+ @remove_tag_prefix = Regexp.new('^' + Regexp.escape(@remove_tag_prefix) + "\\.?")
54
+ end
55
+
56
+ if conf.has_key?('all_segment') && conf['all_segment'].empty?
57
+ @all_segment = true
58
+ end
59
+
60
+ if @all_segment
61
+ @segment_keys = nil
62
+ elsif @segment_keys
63
+ @segment_keys = @segment_keys.strip.split(/\s*,\s*/)
64
+ else
65
+ @segment_keys = []
66
+ end
67
+
68
+ be = BACKENDS[@backend]
69
+ unless be
70
+ raise ConfigError, "unknown backend: #{@backend.inspect}"
71
+ end
72
+
73
+ @backend = be.new
74
+ @backend.configure(conf)
75
+ end
76
+
77
+ def start
78
+ @backend.start
79
+ super
80
+ end
81
+
82
+ def shutdown
83
+ super
84
+ @backend.shutdown
85
+ end
86
+
87
+ def format_stream(tag, es)
88
+ out = ''
89
+ es.each do |time,record|
90
+ value = record[@value_key]
91
+
92
+ fv = value.to_f
93
+ next if fv == 0.0
94
+
95
+ iv = fv.to_i
96
+ if iv.to_f == fv
97
+ value = iv
98
+ else
99
+ value = fv
100
+ end
101
+
102
+ seg_keys = @segment_keys
103
+ unless seg_keys
104
+ seg_keys = record.keys
105
+ seg_keys.delete(@value_key)
106
+ end
107
+
108
+ segs = []
109
+ seg_keys.each {|seg_key|
110
+ if seg_val = record[seg_key]
111
+ segs << seg_key
112
+ segs << seg_val
113
+ end
114
+ }
115
+
116
+ tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
117
+
118
+ [tag, time, value, segs].to_msgpack(out)
119
+ end
120
+ out
121
+ end
122
+
123
+ class SumAggregator
124
+ def initialize
125
+ @value = 0
126
+ end
127
+
128
+ def add(value)
129
+ @value += value
130
+ end
131
+
132
+ attr_reader :value
133
+ end
134
+
135
+ AggregationKey = Struct.new(:tag, :time, :seg_val, :seg_key)
136
+
137
+ def write(chunk)
138
+ counters = {}
139
+
140
+ # select sum(value) from chunk group by tag, time/60, seg_val, seg_key
141
+ chunk.msgpack_each {|tag,time,value,segs|
142
+ time = time / 60 * 60
143
+
144
+ ak = AggregationKey.new(tag, time, nil, nil)
145
+ (counters[ak] ||= SumAggregator.new).add(value)
146
+
147
+ segs.each_slice(2) {|seg_key,seg_val|
148
+ ak = AggregationKey.new(tag, time, seg_val, seg_key)
149
+ (counters[ak] ||= SumAggregator.new).add(value)
150
+ }
151
+ }
152
+
153
+ data = []
154
+ counters.each_pair {|ak,aggr|
155
+ data << [ak.tag, ak.time, aggr.value, ak.seg_key, ak.seg_val]
156
+ }
157
+
158
+ @backend.write(data)
159
+ end
160
+ end
161
+
162
+ end
metadata ADDED
@@ -0,0 +1,119 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-metricsense
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Sadayuki Furuhashi
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-11-08 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: 0.10.6
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: 0.10.6
30
+ - !ruby/object:Gem::Dependency
31
+ name: rake
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: 0.8.7
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: 0.8.7
46
+ - !ruby/object:Gem::Dependency
47
+ name: bundler
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: 1.0.0
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: 1.0.0
62
+ - !ruby/object:Gem::Dependency
63
+ name: simplecov
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: 0.5.4
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: 0.5.4
78
+ description: MetricSense - application metrics aggregation plugin for Fluentd
79
+ email: sf@treasure-data.com
80
+ executables: []
81
+ extensions: []
82
+ extra_rdoc_files: []
83
+ files:
84
+ - .gitignore
85
+ - Gemfile
86
+ - README.rdoc
87
+ - Rakefile
88
+ - VERSION
89
+ - fluent-plugin-metricsense.gemspec
90
+ - lib/fluent/plugin/backend/librato_backend.rb
91
+ - lib/fluent/plugin/backend/rdb_tsdb_backend.rb
92
+ - lib/fluent/plugin/backend/stdout_backend.rb
93
+ - lib/fluent/plugin/out_metricsense.rb
94
+ homepage: https://github.com/treasure-data/fluent-plugin-metricsense
95
+ licenses: []
96
+ post_install_message:
97
+ rdoc_options: []
98
+ require_paths:
99
+ - lib
100
+ required_ruby_version: !ruby/object:Gem::Requirement
101
+ none: false
102
+ requirements:
103
+ - - ! '>='
104
+ - !ruby/object:Gem::Version
105
+ version: '0'
106
+ required_rubygems_version: !ruby/object:Gem::Requirement
107
+ none: false
108
+ requirements:
109
+ - - ! '>='
110
+ - !ruby/object:Gem::Version
111
+ version: '0'
112
+ requirements: []
113
+ rubyforge_project:
114
+ rubygems_version: 1.8.23
115
+ signing_key:
116
+ specification_version: 3
117
+ summary: MetricSense - application metrics aggregation plugin for Fluentd
118
+ test_files: []
119
+ has_rdoc: false