fluent-plugin-metricsense 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/Gemfile +2 -0
- data/README.rdoc +25 -0
- data/Rakefile +17 -0
- data/VERSION +1 -0
- data/fluent-plugin-metricsense.gemspec +22 -0
- data/lib/fluent/plugin/backend/librato_backend.rb +114 -0
- data/lib/fluent/plugin/backend/rdb_tsdb_backend.rb +307 -0
- data/lib/fluent/plugin/backend/stdout_backend.rb +35 -0
- data/lib/fluent/plugin/out_metricsense.rb +162 -0
- metadata +119 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.rdoc
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
= MetricSense - application metrics aggregation plugin for Fluentd
|
2
|
+
|
3
|
+
== Example
|
4
|
+
|
5
|
+
<match metrics.**>
|
6
|
+
type metricsense
|
7
|
+
|
8
|
+
# RDB backend
|
9
|
+
type metricsense
|
10
|
+
backend rdb_tsdb
|
11
|
+
all_segment
|
12
|
+
flush_interval 5m
|
13
|
+
remove_tag_prefix metrics
|
14
|
+
rdb_url mysql2://user:pass@host/mydb
|
15
|
+
rdb_table_prefix ms
|
16
|
+
|
17
|
+
## Librato Metrics backend
|
18
|
+
#all_segment
|
19
|
+
#flush_interval 5m
|
20
|
+
#remove_tag_prefix metrics
|
21
|
+
#backend librato
|
22
|
+
#librato_user xxxx@yyyy
|
23
|
+
#librato_token XYZXYZ
|
24
|
+
</match>
|
25
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'bundler'
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
|
+
|
4
|
+
#require 'rake/testtask'
|
5
|
+
#
|
6
|
+
#Rake::TestTask.new(:test) do |test|
|
7
|
+
# test.libs << 'lib' << 'test'
|
8
|
+
# test.test_files = FileList['test/plugin/*.rb']
|
9
|
+
# test.verbose = true
|
10
|
+
#end
|
11
|
+
|
12
|
+
#task :coverage do |t|
|
13
|
+
# ENV['SIMPLE_COV'] = '1'
|
14
|
+
# Rake::Task["test"].invoke
|
15
|
+
#end
|
16
|
+
|
17
|
+
task :default => [:build]
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,22 @@
|
|
1
|
+
$:.push File.expand_path('../lib', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = "fluent-plugin-metricsense"
|
5
|
+
s.description = "MetricSense - application metrics aggregation plugin for Fluentd"
|
6
|
+
s.summary = s.description
|
7
|
+
s.homepage = "https://github.com/treasure-data/fluent-plugin-metricsense"
|
8
|
+
s.version = File.read("VERSION").strip
|
9
|
+
s.authors = ["Sadayuki Furuhashi"]
|
10
|
+
s.email = "sf@treasure-data.com"
|
11
|
+
s.has_rdoc = false
|
12
|
+
s.require_paths = ['lib']
|
13
|
+
#s.platform = Gem::Platform::RUBY
|
14
|
+
s.files = `git ls-files`.split("\n")
|
15
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
|
+
|
18
|
+
s.add_dependency "fluentd", "~> 0.10.6"
|
19
|
+
s.add_development_dependency "rake", ">= 0.8.7"
|
20
|
+
s.add_development_dependency 'bundler', ['>= 1.0.0']
|
21
|
+
s.add_development_dependency "simplecov", ">= 0.5.4"
|
22
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-metricsense
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 Sadayuki Furuhashi
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
module Fluent::MetricSenseOutput::Backends
|
19
|
+
|
20
|
+
require 'net/http'
|
21
|
+
require 'cgi'
|
22
|
+
require 'json'
|
23
|
+
|
24
|
+
class LibratoBackend < Fluent::MetricSenseOutput::Backend
|
25
|
+
Fluent::MetricSenseOutput::BACKENDS['librato'] = self
|
26
|
+
|
27
|
+
config_param :librato_user, :string
|
28
|
+
config_param :librato_token, :string
|
29
|
+
|
30
|
+
def initialize
|
31
|
+
super
|
32
|
+
@initialized_metrics = {}
|
33
|
+
end
|
34
|
+
|
35
|
+
def write(data)
|
36
|
+
http = Net::HTTP.new('metrics-api.librato.com', 443)
|
37
|
+
http.open_timeout = 60
|
38
|
+
http.use_ssl = true
|
39
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE # FIXME verify
|
40
|
+
#http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
41
|
+
http.cert_store = OpenSSL::X509::Store.new
|
42
|
+
header = {}
|
43
|
+
|
44
|
+
begin
|
45
|
+
# send upto 50 entries at once
|
46
|
+
data.each_slice(50) {|slice|
|
47
|
+
req = Net::HTTP::Post.new('/v1/metrics', header)
|
48
|
+
req.basic_auth @librato_user, @librato_token
|
49
|
+
|
50
|
+
data = []
|
51
|
+
slice.each_with_index {|(tag,time,value,seg_key,seg_val),i|
|
52
|
+
if seg_key
|
53
|
+
name = "#{tag}:#{seg_key}"
|
54
|
+
source = seg_val
|
55
|
+
else
|
56
|
+
name = tag
|
57
|
+
source = nil
|
58
|
+
end
|
59
|
+
h = {
|
60
|
+
"name" => name,
|
61
|
+
"measure_time" => time,
|
62
|
+
"value" => value,
|
63
|
+
}
|
64
|
+
h["source"] = source.to_s if source
|
65
|
+
data << h
|
66
|
+
ensure_metric_initialized(http, name)
|
67
|
+
}
|
68
|
+
body = {"gauges"=>data}.to_json
|
69
|
+
|
70
|
+
$log.trace { "librato metrics: #{data.inspect}" }
|
71
|
+
req.body = body
|
72
|
+
req.set_content_type("application/json")
|
73
|
+
res = http.request(req)
|
74
|
+
|
75
|
+
# TODO error handling
|
76
|
+
if res.code != "200"
|
77
|
+
$log.warn "librato_metrics: #{res.code}: #{res.body}"
|
78
|
+
end
|
79
|
+
}
|
80
|
+
|
81
|
+
ensure
|
82
|
+
http.finish if http.started?
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
METRIC_INITIALIZE_REQUEST = [
|
87
|
+
"type=gauge",
|
88
|
+
"attributes[source_aggregate]=true",
|
89
|
+
"attributes[summarize_function]=sum",
|
90
|
+
"attributes[display_stacked]=true",
|
91
|
+
].join('&')
|
92
|
+
|
93
|
+
def ensure_metric_initialized(http, name)
|
94
|
+
return if @initialized_metrics[name]
|
95
|
+
|
96
|
+
header = {}
|
97
|
+
req = Net::HTTP::Put.new("/v1/metrics/#{CGI.escape name}", header)
|
98
|
+
req.basic_auth @librato_user, @librato_token
|
99
|
+
|
100
|
+
$log.trace { "librato initialize metric: #{name}" }
|
101
|
+
req.body = METRIC_INITIALIZE_REQUEST
|
102
|
+
res = http.request(req)
|
103
|
+
|
104
|
+
# TODO error handling
|
105
|
+
if res.code !~ /20./
|
106
|
+
$log.warn "librato_metrics: #{res.code}: #{res.body}"
|
107
|
+
else
|
108
|
+
@initialized_metrics[name] = true
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
@@ -0,0 +1,307 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-metricsense
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 Sadayuki Furuhashi
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
module Fluent::MetricSenseOutput::Backends
|
19
|
+
|
20
|
+
class RDBTSDBBackend < Fluent::MetricSenseOutput::Backend
|
21
|
+
include Fluent::Configurable
|
22
|
+
|
23
|
+
Fluent::MetricSenseOutput::BACKENDS['rdb_tsdb'] = self
|
24
|
+
|
25
|
+
config_param :rdb_url, :string
|
26
|
+
config_param :rdb_table_prefix, :string
|
27
|
+
config_param :rdb_read_url, :string, :default => nil
|
28
|
+
|
29
|
+
INSERT_SUPPRESS_RING_BUFFER_SIZE = 64
|
30
|
+
|
31
|
+
def initialize
|
32
|
+
super
|
33
|
+
require 'sequel'
|
34
|
+
@insup_ring = []
|
35
|
+
@insup_ring_index = 0
|
36
|
+
end
|
37
|
+
|
38
|
+
def configure(conf)
|
39
|
+
super
|
40
|
+
|
41
|
+
@rdb_read_url ||= @rdb_url
|
42
|
+
|
43
|
+
@metric_tag_table = "#{@rdb_table_prefix}_metric_tags"
|
44
|
+
@segment_value_table = "#{@rdb_table_prefix}_segment_values"
|
45
|
+
@data_table = "#{@rdb_table_prefix}_data"
|
46
|
+
@metric_view = "#{@rdb_table_prefix}_metrics"
|
47
|
+
@metric_json_view = "#{@rdb_table_prefix}_json"
|
48
|
+
|
49
|
+
#sql_standard_concat = lambda {|array| array.join(' || ') }
|
50
|
+
#sql_standard_surround = lambda {|expr| "'\"' || #{expr} || '\"'" }
|
51
|
+
|
52
|
+
case @rdb_url
|
53
|
+
when /^mysql/i
|
54
|
+
@sql_type = :mysql
|
55
|
+
@sql_autoincr_type = "INT"
|
56
|
+
@sql_autoincr_ref_type = "INT"
|
57
|
+
@sql_autoincr_suffix = " AUTO_INCREMENT"
|
58
|
+
@sql_value_type = "SMALLINT"
|
59
|
+
@sql_name_type = "VARCHAR(255)"
|
60
|
+
@sql_time_type = "INT"
|
61
|
+
@sql_insert_ignore = "INSERT IGNORE"
|
62
|
+
@sql_insert_returns_last_id = true
|
63
|
+
#@sql_concat = lambda {|array| "CONCAT(#{array.join(', ')})" }
|
64
|
+
#@sql_surround = lambda {|expr| "CONCAT('\"', #{expr}, '\"')" }
|
65
|
+
when /^postgres/i
|
66
|
+
@sql_type = :postgresql
|
67
|
+
@sql_autoincr_type = "SERIAL"
|
68
|
+
@sql_autoincr_ref_type = "INT"
|
69
|
+
@sql_autoincr_suffix = ""
|
70
|
+
@sql_value_type = "SMALLINT"
|
71
|
+
@sql_name_type = "VARCHAR(255)"
|
72
|
+
@sql_time_type = "INT"
|
73
|
+
@sql_insert_ignore = "INSERT"
|
74
|
+
@sql_insert_returns_last_id = false
|
75
|
+
#@sql_concat = sql_standard_concat
|
76
|
+
#@sql_surround = sql_standard_surround
|
77
|
+
when /^sqlite/i
|
78
|
+
@sql_type = :sqlite
|
79
|
+
@sql_autoincr_type = "INTEGER"
|
80
|
+
@sql_autoincr_ref_type = "INTEGER"
|
81
|
+
@sql_autoincr_suffix = " AUTOINCREMENT"
|
82
|
+
@sql_value_type = "INTEGER"
|
83
|
+
@sql_name_type = "TEXT"
|
84
|
+
@sql_time_type = "INTEGER"
|
85
|
+
@sql_insert_ignore = "INSERT OR IGNORE"
|
86
|
+
@sql_insert_returns_last_id = false
|
87
|
+
#@sql_concat = sql_standard_concat
|
88
|
+
#@sql_surround = sql_standard_surround
|
89
|
+
else
|
90
|
+
@sql_type = :unknown
|
91
|
+
@sql_autoincr_type = "INT"
|
92
|
+
@sql_autoincr_ref_type = "INT"
|
93
|
+
@sql_autoincr_suffix = " AUTO_INCREMENT"
|
94
|
+
@sql_value_type = "SMALLINT"
|
95
|
+
@sql_name_type = "VARCHAR(255)"
|
96
|
+
@sql_time_type = "INT"
|
97
|
+
@sql_insert_ignore = "INSERT IGNORE"
|
98
|
+
@sql_insert_returns_last_id = false
|
99
|
+
#@sql_concat = sql_standard_concat
|
100
|
+
#@sql_surround = sql_standard_surround
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def start
|
105
|
+
ensure_connect do |db|
|
106
|
+
db.run %[
|
107
|
+
CREATE TABLE IF NOT EXISTS `#{@metric_tag_table}` (
|
108
|
+
id #{@sql_autoincr_type} PRIMARY KEY#{@sql_autoincr_suffix},
|
109
|
+
metric_name #{@sql_name_type} NOT NULL,
|
110
|
+
segment_name #{@sql_name_type} NOT NULL,
|
111
|
+
UNIQUE (metric_name, segment_name)
|
112
|
+
);]
|
113
|
+
|
114
|
+
db.run %[
|
115
|
+
CREATE TABLE IF NOT EXISTS `#{@segment_value_table}` (
|
116
|
+
id #{@sql_autoincr_type} PRIMARY KEY#{@sql_autoincr_suffix},
|
117
|
+
name #{@sql_name_type} NOT NULL,
|
118
|
+
UNIQUE (name)
|
119
|
+
);]
|
120
|
+
|
121
|
+
minutes = (0..59).to_a.map {|m| "m#{m} #{@sql_value_type} NOT NULL DEFAULT 0" }.join(', ')
|
122
|
+
db.run %[
|
123
|
+
CREATE TABLE IF NOT EXISTS `#{@data_table}` (
|
124
|
+
base_time #{@sql_time_type} NOT NULL,
|
125
|
+
metric_id #{@sql_autoincr_ref_type} NOT NULL,
|
126
|
+
segment_id #{@sql_autoincr_ref_type},
|
127
|
+
#{minutes},
|
128
|
+
PRIMARY KEY (base_time, metric_id, segment_id)
|
129
|
+
);]
|
130
|
+
|
131
|
+
if @sql_type == :postgresql
|
132
|
+
# ignore duplication error on data_table
|
133
|
+
db.run %[
|
134
|
+
CREATE OR REPLACE RULE ignore_duplicated_insert AS ON INSERT TO `#{@data_table}`
|
135
|
+
WHERE NEW.base_time = OLD.base_time AND NEW.metric_id = OLD.metric_id AND NEW.segment_id = OLD.segment_id
|
136
|
+
DO INSTEAD NOTHING;]
|
137
|
+
end
|
138
|
+
|
139
|
+
#minutes = (0..59).to_a.map {|m| "m#{m}" }.join(', ')
|
140
|
+
#db.run %[
|
141
|
+
# CREATE VIEW IF NOT EXISTS `#{@metric_view}` AS
|
142
|
+
# SELECT
|
143
|
+
# base_time * 60 AS time,
|
144
|
+
# M.metric_name AS metric_name,
|
145
|
+
# CASE M.segment_name WHEN '' THEN NULL ELSE M.segment_name END AS segment_name,
|
146
|
+
# S.name AS segment_value,
|
147
|
+
# #{minutes}
|
148
|
+
# FROM `#{@data_table}` T
|
149
|
+
# LEFT JOIN `#{@metric_tag_table}` M ON T.metric_id = M.id
|
150
|
+
# LEFT JOIN `#{@segment_value_table}` S ON T.segment_id = S.id;]
|
151
|
+
|
152
|
+
#minutes = (0..59).to_a.map {|m| ["m#{m}", "','"] }.flatten!
|
153
|
+
#minutes.pop
|
154
|
+
#minutes = @sql_concat.call(["'['"]+minutes+["']'"])
|
155
|
+
#db.run %[
|
156
|
+
# CREATE VIEW IF NOT EXISTS `#{@metric_json_view}` AS
|
157
|
+
# SELECT
|
158
|
+
# base_time * 60 AS time,
|
159
|
+
# #{@sql_surround.call("M.metric_name")} AS metric_name,
|
160
|
+
# CASE WHEN M.segment_name IS NULL OR M.segment_name = '' THEN 'null' ELSE #{@sql_surround.call("M.segment_name")} END AS segment_name,
|
161
|
+
# CASE WHEN S.name IS NULL OR S.name = '' THEN 'null' ELSE #{@sql_surround.call("S.name")} END AS segment_value,
|
162
|
+
# #{minutes}
|
163
|
+
# FROM `#{@data_table}` T
|
164
|
+
# LEFT JOIN `#{@metric_tag_table}` M ON T.metric_id = M.id
|
165
|
+
# LEFT JOIN `#{@segment_value_table}` S ON T.segment_id = S.id;]
|
166
|
+
|
167
|
+
reload_metric_names!(db)
|
168
|
+
reload_segment_names!(db)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
#def shutdown
|
173
|
+
#end
|
174
|
+
|
175
|
+
ROW_TIME_WINDOW = 60*24
|
176
|
+
ROW_TIME_WINDOW_BITS = 11
|
177
|
+
ROW_TIME_WINDOW_MASK = (1<<ROW_TIME_WINDOW_BITS)-1
|
178
|
+
|
179
|
+
def write(data)
|
180
|
+
ensure_connect do |db|
|
181
|
+
# group by row_key (base_time,metric_id,segment_id)
|
182
|
+
rows = {}
|
183
|
+
data.each {|(tag,time,value,seg_key,seg_val)|
|
184
|
+
base_time = time / ROW_TIME_WINDOW
|
185
|
+
metric_id = get_metric_id(db, tag, seg_key)
|
186
|
+
segment_id = get_segment_id(db, seg_val) if seg_val
|
187
|
+
|
188
|
+
row_key = [base_time, metric_id, segment_id]
|
189
|
+
minutes = (rows[row_key] ||= [])
|
190
|
+
minutes << ((value << ROW_TIME_WINDOW_BITS) | (time % 60))
|
191
|
+
}
|
192
|
+
|
193
|
+
# insert rows if not exist
|
194
|
+
if @sql_type == :sqlite
|
195
|
+
# sqlite3 < 1.3.7 doesn't allow multiple rows at once
|
196
|
+
rows.keys.each {|row_key|
|
197
|
+
next if @insup_ring.include?(row_key)
|
198
|
+
db["#{@sql_insert_ignore} INTO `#{@data_table}` (base_time,metric_id,segment_id) VALUES (?,?,?)", *row_key].update
|
199
|
+
rid = @insup_ring_index = (@insup_ring_index + 1) % INSERT_SUPPRESS_RING_BUFFER_SIZE
|
200
|
+
@insup_ring[rid] = row_key
|
201
|
+
}
|
202
|
+
else
|
203
|
+
insert_sql = "#{@sql_insert_ignore} INTO `#{@data_table}` (base_time,metric_id,segment_id) VALUES " + (["(?,?,?)"] * rows.size).join(', ')
|
204
|
+
insert_params = [insert_sql]
|
205
|
+
rows.keys.each {|row_key|
|
206
|
+
next if @insup_ring.include?(row_key)
|
207
|
+
insert_params.concat(row_key)
|
208
|
+
rid = @insup_ring_index = (@insup_ring_index + 1) % INSERT_SUPPRESS_RING_BUFFER_SIZE
|
209
|
+
@insup_ring[rid] = row_key
|
210
|
+
}
|
211
|
+
db[*insert_params].update
|
212
|
+
end
|
213
|
+
|
214
|
+
# increment values
|
215
|
+
db.transaction do
|
216
|
+
rows.each_pair {|row_key,minutes|
|
217
|
+
update_sql = "UPDATE `#{@data_table}` SET "
|
218
|
+
update_params = [update_sql]
|
219
|
+
|
220
|
+
values_sql = []
|
221
|
+
minutes.each {|m|
|
222
|
+
value = m >> ROW_TIME_WINDOW_BITS
|
223
|
+
minute = m & ROW_TIME_WINDOW_MASK
|
224
|
+
values_sql << "m#{minute} = m#{minute} + ?"
|
225
|
+
update_params << value
|
226
|
+
}.join(', ')
|
227
|
+
update_sql << values_sql.join(', ') << " WHERE base_time=? AND metric_id=? AND segment_id=?"
|
228
|
+
update_params.concat(row_key)
|
229
|
+
|
230
|
+
db[*update_params].update
|
231
|
+
}
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def reload_metric_names!(db)
|
237
|
+
map = {}
|
238
|
+
db.fetch("SELECT id, metric_name, segment_name FROM `#{@metric_tag_table}`") {|row|
|
239
|
+
key = "#{row[:metric_name]}\0#{row[:segment_name]}"
|
240
|
+
map[key] = row[:id]
|
241
|
+
}
|
242
|
+
@metric_names = map
|
243
|
+
end
|
244
|
+
|
245
|
+
def reload_segment_names!(db)
|
246
|
+
map = {}
|
247
|
+
db.fetch("SELECT id, name FROM `#{@segment_value_table}`") {|row|
|
248
|
+
map[row[:name]] = row[:id]
|
249
|
+
}
|
250
|
+
@segment_names = map
|
251
|
+
end
|
252
|
+
|
253
|
+
def get_metric_id(db, tag, seg_key)
|
254
|
+
key = "#{tag}\0#{seg_key}"
|
255
|
+
id = @metric_names[key]
|
256
|
+
return id if id
|
257
|
+
|
258
|
+
begin
|
259
|
+
id = db["INSERT INTO `#{@metric_tag_table}` (metric_name,segment_name) VALUES (?,?)", tag, seg_key||''].update
|
260
|
+
if @sql_insert_returns_last_id
|
261
|
+
@metric_names[key] = id
|
262
|
+
return id
|
263
|
+
end
|
264
|
+
reload_metric_names!(db)
|
265
|
+
return @metric_names[key]
|
266
|
+
|
267
|
+
rescue => e
|
268
|
+
reload_metric_names!(db)
|
269
|
+
id = @metric_names[key]
|
270
|
+
return id if id
|
271
|
+
raise e
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
def get_segment_id(db, seg_val)
|
276
|
+
key = seg_val ? seg_val.to_s : ''
|
277
|
+
id = @segment_names[key]
|
278
|
+
return id if id
|
279
|
+
|
280
|
+
begin
|
281
|
+
id = db["INSERT INTO `#{@segment_value_table}` (name) VALUES (?)", key].update
|
282
|
+
if @sql_insert_returns_last_id
|
283
|
+
@segment_names[key] = id
|
284
|
+
return id
|
285
|
+
end
|
286
|
+
reload_segment_names!(db)
|
287
|
+
return @segment_names[key]
|
288
|
+
|
289
|
+
rescue => e
|
290
|
+
reload_segment_names!(db)
|
291
|
+
id = @segment_names[key]
|
292
|
+
return id if id
|
293
|
+
raise e
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
def ensure_connect(&block)
|
298
|
+
db = Sequel.connect(@rdb_url, :max_connections=>1)
|
299
|
+
begin
|
300
|
+
block.call(db)
|
301
|
+
ensure
|
302
|
+
db.disconnect
|
303
|
+
end
|
304
|
+
end
|
305
|
+
end
|
306
|
+
|
307
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-metricsense
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 Sadayuki Furuhashi
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
module Fluent::MetricSenseOutput::Backends
|
19
|
+
|
20
|
+
class StdoutBackend < Fluent::MetricSenseOutput::Backend
|
21
|
+
Fluent::MetricSenseOutput::BACKENDS['stdout'] = self
|
22
|
+
|
23
|
+
def write(data)
|
24
|
+
data.each {|tag,time,value,seg_key,seg_val|
|
25
|
+
if seg_key
|
26
|
+
puts "#{Time.at(time)} #{tag}: #{value}"
|
27
|
+
else
|
28
|
+
puts "#{Time.at(time)} #{tag} [#{seg_key}=#{seg_val}]: #{value}"
|
29
|
+
end
|
30
|
+
}
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
@@ -0,0 +1,162 @@
|
|
1
|
+
# encoding: ascii-8bit
|
2
|
+
#
|
3
|
+
# fluent-plugin-metricsense
|
4
|
+
#
|
5
|
+
# Copyright (C) 2012 Sadayuki Furuhashi
|
6
|
+
#
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
|
+
# you may not use this file except in compliance with the License.
|
9
|
+
# You may obtain a copy of the License at
|
10
|
+
#
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
12
|
+
#
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
16
|
+
# See the License for the specific language governing permissions and
|
17
|
+
# limitations under the License.
|
18
|
+
#
|
19
|
+
module Fluent
|
20
|
+
|
21
|
+
class MetricSenseOutput < Fluent::BufferedOutput
|
22
|
+
Fluent::Plugin.register_output('metricsense', self)
|
23
|
+
|
24
|
+
BACKENDS = {}
|
25
|
+
|
26
|
+
class Backend
|
27
|
+
include Configurable
|
28
|
+
|
29
|
+
def start
|
30
|
+
end
|
31
|
+
|
32
|
+
def shutdown
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
backend_dir = "#{File.dirname(__FILE__)}/backend"
|
37
|
+
Dir.glob("#{backend_dir}/*_backend.rb") {|e|
|
38
|
+
require e
|
39
|
+
}
|
40
|
+
|
41
|
+
config_param :segment_keys, :string, :default => nil
|
42
|
+
config_param :all_segment, :bool, :default => false
|
43
|
+
config_param :value_key, :string, :default => 'value'
|
44
|
+
|
45
|
+
config_param :backend, :string
|
46
|
+
|
47
|
+
config_param :remove_tag_prefix, :string, :default => nil
|
48
|
+
|
49
|
+
def configure(conf)
|
50
|
+
super
|
51
|
+
|
52
|
+
if @remove_tag_prefix
|
53
|
+
@remove_tag_prefix = Regexp.new('^' + Regexp.escape(@remove_tag_prefix) + "\\.?")
|
54
|
+
end
|
55
|
+
|
56
|
+
if conf.has_key?('all_segment') && conf['all_segment'].empty?
|
57
|
+
@all_segment = true
|
58
|
+
end
|
59
|
+
|
60
|
+
if @all_segment
|
61
|
+
@segment_keys = nil
|
62
|
+
elsif @segment_keys
|
63
|
+
@segment_keys = @segment_keys.strip.split(/\s*,\s*/)
|
64
|
+
else
|
65
|
+
@segment_keys = []
|
66
|
+
end
|
67
|
+
|
68
|
+
be = BACKENDS[@backend]
|
69
|
+
unless be
|
70
|
+
raise ConfigError, "unknown backend: #{@backend.inspect}"
|
71
|
+
end
|
72
|
+
|
73
|
+
@backend = be.new
|
74
|
+
@backend.configure(conf)
|
75
|
+
end
|
76
|
+
|
77
|
+
def start
|
78
|
+
@backend.start
|
79
|
+
super
|
80
|
+
end
|
81
|
+
|
82
|
+
def shutdown
|
83
|
+
super
|
84
|
+
@backend.shutdown
|
85
|
+
end
|
86
|
+
|
87
|
+
def format_stream(tag, es)
|
88
|
+
out = ''
|
89
|
+
es.each do |time,record|
|
90
|
+
value = record[@value_key]
|
91
|
+
|
92
|
+
fv = value.to_f
|
93
|
+
next if fv == 0.0
|
94
|
+
|
95
|
+
iv = fv.to_i
|
96
|
+
if iv.to_f == fv
|
97
|
+
value = iv
|
98
|
+
else
|
99
|
+
value = fv
|
100
|
+
end
|
101
|
+
|
102
|
+
seg_keys = @segment_keys
|
103
|
+
unless seg_keys
|
104
|
+
seg_keys = record.keys
|
105
|
+
seg_keys.delete(@value_key)
|
106
|
+
end
|
107
|
+
|
108
|
+
segs = []
|
109
|
+
seg_keys.each {|seg_key|
|
110
|
+
if seg_val = record[seg_key]
|
111
|
+
segs << seg_key
|
112
|
+
segs << seg_val
|
113
|
+
end
|
114
|
+
}
|
115
|
+
|
116
|
+
tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
|
117
|
+
|
118
|
+
[tag, time, value, segs].to_msgpack(out)
|
119
|
+
end
|
120
|
+
out
|
121
|
+
end
|
122
|
+
|
123
|
+
class SumAggregator
|
124
|
+
def initialize
|
125
|
+
@value = 0
|
126
|
+
end
|
127
|
+
|
128
|
+
def add(value)
|
129
|
+
@value += value
|
130
|
+
end
|
131
|
+
|
132
|
+
attr_reader :value
|
133
|
+
end
|
134
|
+
|
135
|
+
AggregationKey = Struct.new(:tag, :time, :seg_val, :seg_key)
|
136
|
+
|
137
|
+
def write(chunk)
|
138
|
+
counters = {}
|
139
|
+
|
140
|
+
# select sum(value) from chunk group by tag, time/60, seg_val, seg_key
|
141
|
+
chunk.msgpack_each {|tag,time,value,segs|
|
142
|
+
time = time / 60 * 60
|
143
|
+
|
144
|
+
ak = AggregationKey.new(tag, time, nil, nil)
|
145
|
+
(counters[ak] ||= SumAggregator.new).add(value)
|
146
|
+
|
147
|
+
segs.each_slice(2) {|seg_key,seg_val|
|
148
|
+
ak = AggregationKey.new(tag, time, seg_val, seg_key)
|
149
|
+
(counters[ak] ||= SumAggregator.new).add(value)
|
150
|
+
}
|
151
|
+
}
|
152
|
+
|
153
|
+
data = []
|
154
|
+
counters.each_pair {|ak,aggr|
|
155
|
+
data << [ak.tag, ak.time, aggr.value, ak.seg_key, ak.seg_val]
|
156
|
+
}
|
157
|
+
|
158
|
+
@backend.write(data)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
end
|
metadata
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-metricsense
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Sadayuki Furuhashi
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-11-08 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 0.10.6
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.10.6
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 0.8.7
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 0.8.7
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: bundler
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 1.0.0
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.0.0
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: simplecov
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 0.5.4
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 0.5.4
|
78
|
+
description: MetricSense - application metrics aggregation plugin for Fluentd
|
79
|
+
email: sf@treasure-data.com
|
80
|
+
executables: []
|
81
|
+
extensions: []
|
82
|
+
extra_rdoc_files: []
|
83
|
+
files:
|
84
|
+
- .gitignore
|
85
|
+
- Gemfile
|
86
|
+
- README.rdoc
|
87
|
+
- Rakefile
|
88
|
+
- VERSION
|
89
|
+
- fluent-plugin-metricsense.gemspec
|
90
|
+
- lib/fluent/plugin/backend/librato_backend.rb
|
91
|
+
- lib/fluent/plugin/backend/rdb_tsdb_backend.rb
|
92
|
+
- lib/fluent/plugin/backend/stdout_backend.rb
|
93
|
+
- lib/fluent/plugin/out_metricsense.rb
|
94
|
+
homepage: https://github.com/treasure-data/fluent-plugin-metricsense
|
95
|
+
licenses: []
|
96
|
+
post_install_message:
|
97
|
+
rdoc_options: []
|
98
|
+
require_paths:
|
99
|
+
- lib
|
100
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
101
|
+
none: false
|
102
|
+
requirements:
|
103
|
+
- - ! '>='
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
version: '0'
|
106
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
107
|
+
none: false
|
108
|
+
requirements:
|
109
|
+
- - ! '>='
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '0'
|
112
|
+
requirements: []
|
113
|
+
rubyforge_project:
|
114
|
+
rubygems_version: 1.8.23
|
115
|
+
signing_key:
|
116
|
+
specification_version: 3
|
117
|
+
summary: MetricSense - application metrics aggregation plugin for Fluentd
|
118
|
+
test_files: []
|
119
|
+
has_rdoc: false
|