fluent-plugin-metricsense 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/Gemfile +2 -0
- data/README.rdoc +25 -0
- data/Rakefile +17 -0
- data/VERSION +1 -0
- data/fluent-plugin-metricsense.gemspec +22 -0
- data/lib/fluent/plugin/backend/librato_backend.rb +114 -0
- data/lib/fluent/plugin/backend/rdb_tsdb_backend.rb +307 -0
- data/lib/fluent/plugin/backend/stdout_backend.rb +35 -0
- data/lib/fluent/plugin/out_metricsense.rb +162 -0
- metadata +119 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.rdoc
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
= MetricSense - application metrics aggregation plugin for Fluentd
|
|
2
|
+
|
|
3
|
+
== Example
|
|
4
|
+
|
|
5
|
+
<match metrics.**>
|
|
6
|
+
type metricsense
|
|
7
|
+
|
|
8
|
+
# RDB backend
|
|
9
|
+
type metricsense
|
|
10
|
+
backend rdb_tsdb
|
|
11
|
+
all_segment
|
|
12
|
+
flush_interval 5m
|
|
13
|
+
remove_tag_prefix metrics
|
|
14
|
+
rdb_url mysql2://user:pass@host/mydb
|
|
15
|
+
rdb_table_prefix ms
|
|
16
|
+
|
|
17
|
+
## Librato Metrics backend
|
|
18
|
+
#all_segment
|
|
19
|
+
#flush_interval 5m
|
|
20
|
+
#remove_tag_prefix metrics
|
|
21
|
+
#backend librato
|
|
22
|
+
#librato_user xxxx@yyyy
|
|
23
|
+
#librato_token XYZXYZ
|
|
24
|
+
</match>
|
|
25
|
+
|
data/Rakefile
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
require 'bundler'
|
|
2
|
+
Bundler::GemHelper.install_tasks
|
|
3
|
+
|
|
4
|
+
#require 'rake/testtask'
|
|
5
|
+
#
|
|
6
|
+
#Rake::TestTask.new(:test) do |test|
|
|
7
|
+
# test.libs << 'lib' << 'test'
|
|
8
|
+
# test.test_files = FileList['test/plugin/*.rb']
|
|
9
|
+
# test.verbose = true
|
|
10
|
+
#end
|
|
11
|
+
|
|
12
|
+
#task :coverage do |t|
|
|
13
|
+
# ENV['SIMPLE_COV'] = '1'
|
|
14
|
+
# Rake::Task["test"].invoke
|
|
15
|
+
#end
|
|
16
|
+
|
|
17
|
+
task :default => [:build]
|
data/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.1.0
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
$:.push File.expand_path('../lib', __FILE__)
|
|
2
|
+
|
|
3
|
+
Gem::Specification.new do |s|
|
|
4
|
+
s.name = "fluent-plugin-metricsense"
|
|
5
|
+
s.description = "MetricSense - application metrics aggregation plugin for Fluentd"
|
|
6
|
+
s.summary = s.description
|
|
7
|
+
s.homepage = "https://github.com/treasure-data/fluent-plugin-metricsense"
|
|
8
|
+
s.version = File.read("VERSION").strip
|
|
9
|
+
s.authors = ["Sadayuki Furuhashi"]
|
|
10
|
+
s.email = "sf@treasure-data.com"
|
|
11
|
+
s.has_rdoc = false
|
|
12
|
+
s.require_paths = ['lib']
|
|
13
|
+
#s.platform = Gem::Platform::RUBY
|
|
14
|
+
s.files = `git ls-files`.split("\n")
|
|
15
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
|
16
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
|
17
|
+
|
|
18
|
+
s.add_dependency "fluentd", "~> 0.10.6"
|
|
19
|
+
s.add_development_dependency "rake", ">= 0.8.7"
|
|
20
|
+
s.add_development_dependency 'bundler', ['>= 1.0.0']
|
|
21
|
+
s.add_development_dependency "simplecov", ">= 0.5.4"
|
|
22
|
+
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
#
|
|
2
|
+
# fluent-plugin-metricsense
|
|
3
|
+
#
|
|
4
|
+
# Copyright (C) 2012 Sadayuki Furuhashi
|
|
5
|
+
#
|
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License.
|
|
8
|
+
# You may obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
# See the License for the specific language governing permissions and
|
|
16
|
+
# limitations under the License.
|
|
17
|
+
#
|
|
18
|
+
module Fluent::MetricSenseOutput::Backends
|
|
19
|
+
|
|
20
|
+
require 'net/http'
|
|
21
|
+
require 'cgi'
|
|
22
|
+
require 'json'
|
|
23
|
+
|
|
24
|
+
class LibratoBackend < Fluent::MetricSenseOutput::Backend
|
|
25
|
+
Fluent::MetricSenseOutput::BACKENDS['librato'] = self
|
|
26
|
+
|
|
27
|
+
config_param :librato_user, :string
|
|
28
|
+
config_param :librato_token, :string
|
|
29
|
+
|
|
30
|
+
def initialize
|
|
31
|
+
super
|
|
32
|
+
@initialized_metrics = {}
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def write(data)
|
|
36
|
+
http = Net::HTTP.new('metrics-api.librato.com', 443)
|
|
37
|
+
http.open_timeout = 60
|
|
38
|
+
http.use_ssl = true
|
|
39
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE # FIXME verify
|
|
40
|
+
#http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
|
41
|
+
http.cert_store = OpenSSL::X509::Store.new
|
|
42
|
+
header = {}
|
|
43
|
+
|
|
44
|
+
begin
|
|
45
|
+
# send upto 50 entries at once
|
|
46
|
+
data.each_slice(50) {|slice|
|
|
47
|
+
req = Net::HTTP::Post.new('/v1/metrics', header)
|
|
48
|
+
req.basic_auth @librato_user, @librato_token
|
|
49
|
+
|
|
50
|
+
data = []
|
|
51
|
+
slice.each_with_index {|(tag,time,value,seg_key,seg_val),i|
|
|
52
|
+
if seg_key
|
|
53
|
+
name = "#{tag}:#{seg_key}"
|
|
54
|
+
source = seg_val
|
|
55
|
+
else
|
|
56
|
+
name = tag
|
|
57
|
+
source = nil
|
|
58
|
+
end
|
|
59
|
+
h = {
|
|
60
|
+
"name" => name,
|
|
61
|
+
"measure_time" => time,
|
|
62
|
+
"value" => value,
|
|
63
|
+
}
|
|
64
|
+
h["source"] = source.to_s if source
|
|
65
|
+
data << h
|
|
66
|
+
ensure_metric_initialized(http, name)
|
|
67
|
+
}
|
|
68
|
+
body = {"gauges"=>data}.to_json
|
|
69
|
+
|
|
70
|
+
$log.trace { "librato metrics: #{data.inspect}" }
|
|
71
|
+
req.body = body
|
|
72
|
+
req.set_content_type("application/json")
|
|
73
|
+
res = http.request(req)
|
|
74
|
+
|
|
75
|
+
# TODO error handling
|
|
76
|
+
if res.code != "200"
|
|
77
|
+
$log.warn "librato_metrics: #{res.code}: #{res.body}"
|
|
78
|
+
end
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
ensure
|
|
82
|
+
http.finish if http.started?
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
METRIC_INITIALIZE_REQUEST = [
|
|
87
|
+
"type=gauge",
|
|
88
|
+
"attributes[source_aggregate]=true",
|
|
89
|
+
"attributes[summarize_function]=sum",
|
|
90
|
+
"attributes[display_stacked]=true",
|
|
91
|
+
].join('&')
|
|
92
|
+
|
|
93
|
+
def ensure_metric_initialized(http, name)
|
|
94
|
+
return if @initialized_metrics[name]
|
|
95
|
+
|
|
96
|
+
header = {}
|
|
97
|
+
req = Net::HTTP::Put.new("/v1/metrics/#{CGI.escape name}", header)
|
|
98
|
+
req.basic_auth @librato_user, @librato_token
|
|
99
|
+
|
|
100
|
+
$log.trace { "librato initialize metric: #{name}" }
|
|
101
|
+
req.body = METRIC_INITIALIZE_REQUEST
|
|
102
|
+
res = http.request(req)
|
|
103
|
+
|
|
104
|
+
# TODO error handling
|
|
105
|
+
if res.code !~ /20./
|
|
106
|
+
$log.warn "librato_metrics: #{res.code}: #{res.body}"
|
|
107
|
+
else
|
|
108
|
+
@initialized_metrics[name] = true
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
end
|
|
114
|
+
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
#
|
|
2
|
+
# fluent-plugin-metricsense
|
|
3
|
+
#
|
|
4
|
+
# Copyright (C) 2012 Sadayuki Furuhashi
|
|
5
|
+
#
|
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License.
|
|
8
|
+
# You may obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
# See the License for the specific language governing permissions and
|
|
16
|
+
# limitations under the License.
|
|
17
|
+
#
|
|
18
|
+
module Fluent::MetricSenseOutput::Backends
|
|
19
|
+
|
|
20
|
+
class RDBTSDBBackend < Fluent::MetricSenseOutput::Backend
|
|
21
|
+
include Fluent::Configurable
|
|
22
|
+
|
|
23
|
+
Fluent::MetricSenseOutput::BACKENDS['rdb_tsdb'] = self
|
|
24
|
+
|
|
25
|
+
config_param :rdb_url, :string
|
|
26
|
+
config_param :rdb_table_prefix, :string
|
|
27
|
+
config_param :rdb_read_url, :string, :default => nil
|
|
28
|
+
|
|
29
|
+
INSERT_SUPPRESS_RING_BUFFER_SIZE = 64
|
|
30
|
+
|
|
31
|
+
def initialize
|
|
32
|
+
super
|
|
33
|
+
require 'sequel'
|
|
34
|
+
@insup_ring = []
|
|
35
|
+
@insup_ring_index = 0
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def configure(conf)
|
|
39
|
+
super
|
|
40
|
+
|
|
41
|
+
@rdb_read_url ||= @rdb_url
|
|
42
|
+
|
|
43
|
+
@metric_tag_table = "#{@rdb_table_prefix}_metric_tags"
|
|
44
|
+
@segment_value_table = "#{@rdb_table_prefix}_segment_values"
|
|
45
|
+
@data_table = "#{@rdb_table_prefix}_data"
|
|
46
|
+
@metric_view = "#{@rdb_table_prefix}_metrics"
|
|
47
|
+
@metric_json_view = "#{@rdb_table_prefix}_json"
|
|
48
|
+
|
|
49
|
+
#sql_standard_concat = lambda {|array| array.join(' || ') }
|
|
50
|
+
#sql_standard_surround = lambda {|expr| "'\"' || #{expr} || '\"'" }
|
|
51
|
+
|
|
52
|
+
case @rdb_url
|
|
53
|
+
when /^mysql/i
|
|
54
|
+
@sql_type = :mysql
|
|
55
|
+
@sql_autoincr_type = "INT"
|
|
56
|
+
@sql_autoincr_ref_type = "INT"
|
|
57
|
+
@sql_autoincr_suffix = " AUTO_INCREMENT"
|
|
58
|
+
@sql_value_type = "SMALLINT"
|
|
59
|
+
@sql_name_type = "VARCHAR(255)"
|
|
60
|
+
@sql_time_type = "INT"
|
|
61
|
+
@sql_insert_ignore = "INSERT IGNORE"
|
|
62
|
+
@sql_insert_returns_last_id = true
|
|
63
|
+
#@sql_concat = lambda {|array| "CONCAT(#{array.join(', ')})" }
|
|
64
|
+
#@sql_surround = lambda {|expr| "CONCAT('\"', #{expr}, '\"')" }
|
|
65
|
+
when /^postgres/i
|
|
66
|
+
@sql_type = :postgresql
|
|
67
|
+
@sql_autoincr_type = "SERIAL"
|
|
68
|
+
@sql_autoincr_ref_type = "INT"
|
|
69
|
+
@sql_autoincr_suffix = ""
|
|
70
|
+
@sql_value_type = "SMALLINT"
|
|
71
|
+
@sql_name_type = "VARCHAR(255)"
|
|
72
|
+
@sql_time_type = "INT"
|
|
73
|
+
@sql_insert_ignore = "INSERT"
|
|
74
|
+
@sql_insert_returns_last_id = false
|
|
75
|
+
#@sql_concat = sql_standard_concat
|
|
76
|
+
#@sql_surround = sql_standard_surround
|
|
77
|
+
when /^sqlite/i
|
|
78
|
+
@sql_type = :sqlite
|
|
79
|
+
@sql_autoincr_type = "INTEGER"
|
|
80
|
+
@sql_autoincr_ref_type = "INTEGER"
|
|
81
|
+
@sql_autoincr_suffix = " AUTOINCREMENT"
|
|
82
|
+
@sql_value_type = "INTEGER"
|
|
83
|
+
@sql_name_type = "TEXT"
|
|
84
|
+
@sql_time_type = "INTEGER"
|
|
85
|
+
@sql_insert_ignore = "INSERT OR IGNORE"
|
|
86
|
+
@sql_insert_returns_last_id = false
|
|
87
|
+
#@sql_concat = sql_standard_concat
|
|
88
|
+
#@sql_surround = sql_standard_surround
|
|
89
|
+
else
|
|
90
|
+
@sql_type = :unknown
|
|
91
|
+
@sql_autoincr_type = "INT"
|
|
92
|
+
@sql_autoincr_ref_type = "INT"
|
|
93
|
+
@sql_autoincr_suffix = " AUTO_INCREMENT"
|
|
94
|
+
@sql_value_type = "SMALLINT"
|
|
95
|
+
@sql_name_type = "VARCHAR(255)"
|
|
96
|
+
@sql_time_type = "INT"
|
|
97
|
+
@sql_insert_ignore = "INSERT IGNORE"
|
|
98
|
+
@sql_insert_returns_last_id = false
|
|
99
|
+
#@sql_concat = sql_standard_concat
|
|
100
|
+
#@sql_surround = sql_standard_surround
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def start
|
|
105
|
+
ensure_connect do |db|
|
|
106
|
+
db.run %[
|
|
107
|
+
CREATE TABLE IF NOT EXISTS `#{@metric_tag_table}` (
|
|
108
|
+
id #{@sql_autoincr_type} PRIMARY KEY#{@sql_autoincr_suffix},
|
|
109
|
+
metric_name #{@sql_name_type} NOT NULL,
|
|
110
|
+
segment_name #{@sql_name_type} NOT NULL,
|
|
111
|
+
UNIQUE (metric_name, segment_name)
|
|
112
|
+
);]
|
|
113
|
+
|
|
114
|
+
db.run %[
|
|
115
|
+
CREATE TABLE IF NOT EXISTS `#{@segment_value_table}` (
|
|
116
|
+
id #{@sql_autoincr_type} PRIMARY KEY#{@sql_autoincr_suffix},
|
|
117
|
+
name #{@sql_name_type} NOT NULL,
|
|
118
|
+
UNIQUE (name)
|
|
119
|
+
);]
|
|
120
|
+
|
|
121
|
+
minutes = (0..59).to_a.map {|m| "m#{m} #{@sql_value_type} NOT NULL DEFAULT 0" }.join(', ')
|
|
122
|
+
db.run %[
|
|
123
|
+
CREATE TABLE IF NOT EXISTS `#{@data_table}` (
|
|
124
|
+
base_time #{@sql_time_type} NOT NULL,
|
|
125
|
+
metric_id #{@sql_autoincr_ref_type} NOT NULL,
|
|
126
|
+
segment_id #{@sql_autoincr_ref_type},
|
|
127
|
+
#{minutes},
|
|
128
|
+
PRIMARY KEY (base_time, metric_id, segment_id)
|
|
129
|
+
);]
|
|
130
|
+
|
|
131
|
+
if @sql_type == :postgresql
|
|
132
|
+
# ignore duplication error on data_table
|
|
133
|
+
db.run %[
|
|
134
|
+
CREATE OR REPLACE RULE ignore_duplicated_insert AS ON INSERT TO `#{@data_table}`
|
|
135
|
+
WHERE NEW.base_time = OLD.base_time AND NEW.metric_id = OLD.metric_id AND NEW.segment_id = OLD.segment_id
|
|
136
|
+
DO INSTEAD NOTHING;]
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
#minutes = (0..59).to_a.map {|m| "m#{m}" }.join(', ')
|
|
140
|
+
#db.run %[
|
|
141
|
+
# CREATE VIEW IF NOT EXISTS `#{@metric_view}` AS
|
|
142
|
+
# SELECT
|
|
143
|
+
# base_time * 60 AS time,
|
|
144
|
+
# M.metric_name AS metric_name,
|
|
145
|
+
# CASE M.segment_name WHEN '' THEN NULL ELSE M.segment_name END AS segment_name,
|
|
146
|
+
# S.name AS segment_value,
|
|
147
|
+
# #{minutes}
|
|
148
|
+
# FROM `#{@data_table}` T
|
|
149
|
+
# LEFT JOIN `#{@metric_tag_table}` M ON T.metric_id = M.id
|
|
150
|
+
# LEFT JOIN `#{@segment_value_table}` S ON T.segment_id = S.id;]
|
|
151
|
+
|
|
152
|
+
#minutes = (0..59).to_a.map {|m| ["m#{m}", "','"] }.flatten!
|
|
153
|
+
#minutes.pop
|
|
154
|
+
#minutes = @sql_concat.call(["'['"]+minutes+["']'"])
|
|
155
|
+
#db.run %[
|
|
156
|
+
# CREATE VIEW IF NOT EXISTS `#{@metric_json_view}` AS
|
|
157
|
+
# SELECT
|
|
158
|
+
# base_time * 60 AS time,
|
|
159
|
+
# #{@sql_surround.call("M.metric_name")} AS metric_name,
|
|
160
|
+
# CASE WHEN M.segment_name IS NULL OR M.segment_name = '' THEN 'null' ELSE #{@sql_surround.call("M.segment_name")} END AS segment_name,
|
|
161
|
+
# CASE WHEN S.name IS NULL OR S.name = '' THEN 'null' ELSE #{@sql_surround.call("S.name")} END AS segment_value,
|
|
162
|
+
# #{minutes}
|
|
163
|
+
# FROM `#{@data_table}` T
|
|
164
|
+
# LEFT JOIN `#{@metric_tag_table}` M ON T.metric_id = M.id
|
|
165
|
+
# LEFT JOIN `#{@segment_value_table}` S ON T.segment_id = S.id;]
|
|
166
|
+
|
|
167
|
+
reload_metric_names!(db)
|
|
168
|
+
reload_segment_names!(db)
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
#def shutdown
|
|
173
|
+
#end
|
|
174
|
+
|
|
175
|
+
ROW_TIME_WINDOW = 60*24
|
|
176
|
+
ROW_TIME_WINDOW_BITS = 11
|
|
177
|
+
ROW_TIME_WINDOW_MASK = (1<<ROW_TIME_WINDOW_BITS)-1
|
|
178
|
+
|
|
179
|
+
def write(data)
|
|
180
|
+
ensure_connect do |db|
|
|
181
|
+
# group by row_key (base_time,metric_id,segment_id)
|
|
182
|
+
rows = {}
|
|
183
|
+
data.each {|(tag,time,value,seg_key,seg_val)|
|
|
184
|
+
base_time = time / ROW_TIME_WINDOW
|
|
185
|
+
metric_id = get_metric_id(db, tag, seg_key)
|
|
186
|
+
segment_id = get_segment_id(db, seg_val) if seg_val
|
|
187
|
+
|
|
188
|
+
row_key = [base_time, metric_id, segment_id]
|
|
189
|
+
minutes = (rows[row_key] ||= [])
|
|
190
|
+
minutes << ((value << ROW_TIME_WINDOW_BITS) | (time % 60))
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
# insert rows if not exist
|
|
194
|
+
if @sql_type == :sqlite
|
|
195
|
+
# sqlite3 < 1.3.7 doesn't allow multiple rows at once
|
|
196
|
+
rows.keys.each {|row_key|
|
|
197
|
+
next if @insup_ring.include?(row_key)
|
|
198
|
+
db["#{@sql_insert_ignore} INTO `#{@data_table}` (base_time,metric_id,segment_id) VALUES (?,?,?)", *row_key].update
|
|
199
|
+
rid = @insup_ring_index = (@insup_ring_index + 1) % INSERT_SUPPRESS_RING_BUFFER_SIZE
|
|
200
|
+
@insup_ring[rid] = row_key
|
|
201
|
+
}
|
|
202
|
+
else
|
|
203
|
+
insert_sql = "#{@sql_insert_ignore} INTO `#{@data_table}` (base_time,metric_id,segment_id) VALUES " + (["(?,?,?)"] * rows.size).join(', ')
|
|
204
|
+
insert_params = [insert_sql]
|
|
205
|
+
rows.keys.each {|row_key|
|
|
206
|
+
next if @insup_ring.include?(row_key)
|
|
207
|
+
insert_params.concat(row_key)
|
|
208
|
+
rid = @insup_ring_index = (@insup_ring_index + 1) % INSERT_SUPPRESS_RING_BUFFER_SIZE
|
|
209
|
+
@insup_ring[rid] = row_key
|
|
210
|
+
}
|
|
211
|
+
db[*insert_params].update
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# increment values
|
|
215
|
+
db.transaction do
|
|
216
|
+
rows.each_pair {|row_key,minutes|
|
|
217
|
+
update_sql = "UPDATE `#{@data_table}` SET "
|
|
218
|
+
update_params = [update_sql]
|
|
219
|
+
|
|
220
|
+
values_sql = []
|
|
221
|
+
minutes.each {|m|
|
|
222
|
+
value = m >> ROW_TIME_WINDOW_BITS
|
|
223
|
+
minute = m & ROW_TIME_WINDOW_MASK
|
|
224
|
+
values_sql << "m#{minute} = m#{minute} + ?"
|
|
225
|
+
update_params << value
|
|
226
|
+
}.join(', ')
|
|
227
|
+
update_sql << values_sql.join(', ') << " WHERE base_time=? AND metric_id=? AND segment_id=?"
|
|
228
|
+
update_params.concat(row_key)
|
|
229
|
+
|
|
230
|
+
db[*update_params].update
|
|
231
|
+
}
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def reload_metric_names!(db)
|
|
237
|
+
map = {}
|
|
238
|
+
db.fetch("SELECT id, metric_name, segment_name FROM `#{@metric_tag_table}`") {|row|
|
|
239
|
+
key = "#{row[:metric_name]}\0#{row[:segment_name]}"
|
|
240
|
+
map[key] = row[:id]
|
|
241
|
+
}
|
|
242
|
+
@metric_names = map
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def reload_segment_names!(db)
|
|
246
|
+
map = {}
|
|
247
|
+
db.fetch("SELECT id, name FROM `#{@segment_value_table}`") {|row|
|
|
248
|
+
map[row[:name]] = row[:id]
|
|
249
|
+
}
|
|
250
|
+
@segment_names = map
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def get_metric_id(db, tag, seg_key)
|
|
254
|
+
key = "#{tag}\0#{seg_key}"
|
|
255
|
+
id = @metric_names[key]
|
|
256
|
+
return id if id
|
|
257
|
+
|
|
258
|
+
begin
|
|
259
|
+
id = db["INSERT INTO `#{@metric_tag_table}` (metric_name,segment_name) VALUES (?,?)", tag, seg_key||''].update
|
|
260
|
+
if @sql_insert_returns_last_id
|
|
261
|
+
@metric_names[key] = id
|
|
262
|
+
return id
|
|
263
|
+
end
|
|
264
|
+
reload_metric_names!(db)
|
|
265
|
+
return @metric_names[key]
|
|
266
|
+
|
|
267
|
+
rescue => e
|
|
268
|
+
reload_metric_names!(db)
|
|
269
|
+
id = @metric_names[key]
|
|
270
|
+
return id if id
|
|
271
|
+
raise e
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def get_segment_id(db, seg_val)
|
|
276
|
+
key = seg_val ? seg_val.to_s : ''
|
|
277
|
+
id = @segment_names[key]
|
|
278
|
+
return id if id
|
|
279
|
+
|
|
280
|
+
begin
|
|
281
|
+
id = db["INSERT INTO `#{@segment_value_table}` (name) VALUES (?)", key].update
|
|
282
|
+
if @sql_insert_returns_last_id
|
|
283
|
+
@segment_names[key] = id
|
|
284
|
+
return id
|
|
285
|
+
end
|
|
286
|
+
reload_segment_names!(db)
|
|
287
|
+
return @segment_names[key]
|
|
288
|
+
|
|
289
|
+
rescue => e
|
|
290
|
+
reload_segment_names!(db)
|
|
291
|
+
id = @segment_names[key]
|
|
292
|
+
return id if id
|
|
293
|
+
raise e
|
|
294
|
+
end
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def ensure_connect(&block)
|
|
298
|
+
db = Sequel.connect(@rdb_url, :max_connections=>1)
|
|
299
|
+
begin
|
|
300
|
+
block.call(db)
|
|
301
|
+
ensure
|
|
302
|
+
db.disconnect
|
|
303
|
+
end
|
|
304
|
+
end
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#
|
|
2
|
+
# fluent-plugin-metricsense
|
|
3
|
+
#
|
|
4
|
+
# Copyright (C) 2012 Sadayuki Furuhashi
|
|
5
|
+
#
|
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License.
|
|
8
|
+
# You may obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
# See the License for the specific language governing permissions and
|
|
16
|
+
# limitations under the License.
|
|
17
|
+
#
|
|
18
|
+
module Fluent::MetricSenseOutput::Backends
|
|
19
|
+
|
|
20
|
+
class StdoutBackend < Fluent::MetricSenseOutput::Backend
|
|
21
|
+
Fluent::MetricSenseOutput::BACKENDS['stdout'] = self
|
|
22
|
+
|
|
23
|
+
def write(data)
|
|
24
|
+
data.each {|tag,time,value,seg_key,seg_val|
|
|
25
|
+
if seg_key
|
|
26
|
+
puts "#{Time.at(time)} #{tag}: #{value}"
|
|
27
|
+
else
|
|
28
|
+
puts "#{Time.at(time)} #{tag} [#{seg_key}=#{seg_val}]: #{value}"
|
|
29
|
+
end
|
|
30
|
+
}
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
end
|
|
35
|
+
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# encoding: ascii-8bit
|
|
2
|
+
#
|
|
3
|
+
# fluent-plugin-metricsense
|
|
4
|
+
#
|
|
5
|
+
# Copyright (C) 2012 Sadayuki Furuhashi
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
#
|
|
19
|
+
module Fluent
|
|
20
|
+
|
|
21
|
+
class MetricSenseOutput < Fluent::BufferedOutput
|
|
22
|
+
Fluent::Plugin.register_output('metricsense', self)
|
|
23
|
+
|
|
24
|
+
BACKENDS = {}
|
|
25
|
+
|
|
26
|
+
class Backend
|
|
27
|
+
include Configurable
|
|
28
|
+
|
|
29
|
+
def start
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def shutdown
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
backend_dir = "#{File.dirname(__FILE__)}/backend"
|
|
37
|
+
Dir.glob("#{backend_dir}/*_backend.rb") {|e|
|
|
38
|
+
require e
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
config_param :segment_keys, :string, :default => nil
|
|
42
|
+
config_param :all_segment, :bool, :default => false
|
|
43
|
+
config_param :value_key, :string, :default => 'value'
|
|
44
|
+
|
|
45
|
+
config_param :backend, :string
|
|
46
|
+
|
|
47
|
+
config_param :remove_tag_prefix, :string, :default => nil
|
|
48
|
+
|
|
49
|
+
def configure(conf)
|
|
50
|
+
super
|
|
51
|
+
|
|
52
|
+
if @remove_tag_prefix
|
|
53
|
+
@remove_tag_prefix = Regexp.new('^' + Regexp.escape(@remove_tag_prefix) + "\\.?")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
if conf.has_key?('all_segment') && conf['all_segment'].empty?
|
|
57
|
+
@all_segment = true
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
if @all_segment
|
|
61
|
+
@segment_keys = nil
|
|
62
|
+
elsif @segment_keys
|
|
63
|
+
@segment_keys = @segment_keys.strip.split(/\s*,\s*/)
|
|
64
|
+
else
|
|
65
|
+
@segment_keys = []
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
be = BACKENDS[@backend]
|
|
69
|
+
unless be
|
|
70
|
+
raise ConfigError, "unknown backend: #{@backend.inspect}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
@backend = be.new
|
|
74
|
+
@backend.configure(conf)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def start
|
|
78
|
+
@backend.start
|
|
79
|
+
super
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def shutdown
|
|
83
|
+
super
|
|
84
|
+
@backend.shutdown
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def format_stream(tag, es)
|
|
88
|
+
out = ''
|
|
89
|
+
es.each do |time,record|
|
|
90
|
+
value = record[@value_key]
|
|
91
|
+
|
|
92
|
+
fv = value.to_f
|
|
93
|
+
next if fv == 0.0
|
|
94
|
+
|
|
95
|
+
iv = fv.to_i
|
|
96
|
+
if iv.to_f == fv
|
|
97
|
+
value = iv
|
|
98
|
+
else
|
|
99
|
+
value = fv
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
seg_keys = @segment_keys
|
|
103
|
+
unless seg_keys
|
|
104
|
+
seg_keys = record.keys
|
|
105
|
+
seg_keys.delete(@value_key)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
segs = []
|
|
109
|
+
seg_keys.each {|seg_key|
|
|
110
|
+
if seg_val = record[seg_key]
|
|
111
|
+
segs << seg_key
|
|
112
|
+
segs << seg_val
|
|
113
|
+
end
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
|
|
117
|
+
|
|
118
|
+
[tag, time, value, segs].to_msgpack(out)
|
|
119
|
+
end
|
|
120
|
+
out
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
class SumAggregator
|
|
124
|
+
def initialize
|
|
125
|
+
@value = 0
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def add(value)
|
|
129
|
+
@value += value
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
attr_reader :value
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
AggregationKey = Struct.new(:tag, :time, :seg_val, :seg_key)
|
|
136
|
+
|
|
137
|
+
def write(chunk)
|
|
138
|
+
counters = {}
|
|
139
|
+
|
|
140
|
+
# select sum(value) from chunk group by tag, time/60, seg_val, seg_key
|
|
141
|
+
chunk.msgpack_each {|tag,time,value,segs|
|
|
142
|
+
time = time / 60 * 60
|
|
143
|
+
|
|
144
|
+
ak = AggregationKey.new(tag, time, nil, nil)
|
|
145
|
+
(counters[ak] ||= SumAggregator.new).add(value)
|
|
146
|
+
|
|
147
|
+
segs.each_slice(2) {|seg_key,seg_val|
|
|
148
|
+
ak = AggregationKey.new(tag, time, seg_val, seg_key)
|
|
149
|
+
(counters[ak] ||= SumAggregator.new).add(value)
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
data = []
|
|
154
|
+
counters.each_pair {|ak,aggr|
|
|
155
|
+
data << [ak.tag, ak.time, aggr.value, ak.seg_key, ak.seg_val]
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
@backend.write(data)
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: fluent-plugin-metricsense
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
prerelease:
|
|
6
|
+
platform: ruby
|
|
7
|
+
authors:
|
|
8
|
+
- Sadayuki Furuhashi
|
|
9
|
+
autorequire:
|
|
10
|
+
bindir: bin
|
|
11
|
+
cert_chain: []
|
|
12
|
+
date: 2012-11-08 00:00:00.000000000 Z
|
|
13
|
+
dependencies:
|
|
14
|
+
- !ruby/object:Gem::Dependency
|
|
15
|
+
name: fluentd
|
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
|
17
|
+
none: false
|
|
18
|
+
requirements:
|
|
19
|
+
- - ~>
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: 0.10.6
|
|
22
|
+
type: :runtime
|
|
23
|
+
prerelease: false
|
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
25
|
+
none: false
|
|
26
|
+
requirements:
|
|
27
|
+
- - ~>
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: 0.10.6
|
|
30
|
+
- !ruby/object:Gem::Dependency
|
|
31
|
+
name: rake
|
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
|
33
|
+
none: false
|
|
34
|
+
requirements:
|
|
35
|
+
- - ! '>='
|
|
36
|
+
- !ruby/object:Gem::Version
|
|
37
|
+
version: 0.8.7
|
|
38
|
+
type: :development
|
|
39
|
+
prerelease: false
|
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
41
|
+
none: false
|
|
42
|
+
requirements:
|
|
43
|
+
- - ! '>='
|
|
44
|
+
- !ruby/object:Gem::Version
|
|
45
|
+
version: 0.8.7
|
|
46
|
+
- !ruby/object:Gem::Dependency
|
|
47
|
+
name: bundler
|
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
|
49
|
+
none: false
|
|
50
|
+
requirements:
|
|
51
|
+
- - ! '>='
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: 1.0.0
|
|
54
|
+
type: :development
|
|
55
|
+
prerelease: false
|
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
57
|
+
none: false
|
|
58
|
+
requirements:
|
|
59
|
+
- - ! '>='
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 1.0.0
|
|
62
|
+
- !ruby/object:Gem::Dependency
|
|
63
|
+
name: simplecov
|
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
|
65
|
+
none: false
|
|
66
|
+
requirements:
|
|
67
|
+
- - ! '>='
|
|
68
|
+
- !ruby/object:Gem::Version
|
|
69
|
+
version: 0.5.4
|
|
70
|
+
type: :development
|
|
71
|
+
prerelease: false
|
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
73
|
+
none: false
|
|
74
|
+
requirements:
|
|
75
|
+
- - ! '>='
|
|
76
|
+
- !ruby/object:Gem::Version
|
|
77
|
+
version: 0.5.4
|
|
78
|
+
description: MetricSense - application metrics aggregation plugin for Fluentd
|
|
79
|
+
email: sf@treasure-data.com
|
|
80
|
+
executables: []
|
|
81
|
+
extensions: []
|
|
82
|
+
extra_rdoc_files: []
|
|
83
|
+
files:
|
|
84
|
+
- .gitignore
|
|
85
|
+
- Gemfile
|
|
86
|
+
- README.rdoc
|
|
87
|
+
- Rakefile
|
|
88
|
+
- VERSION
|
|
89
|
+
- fluent-plugin-metricsense.gemspec
|
|
90
|
+
- lib/fluent/plugin/backend/librato_backend.rb
|
|
91
|
+
- lib/fluent/plugin/backend/rdb_tsdb_backend.rb
|
|
92
|
+
- lib/fluent/plugin/backend/stdout_backend.rb
|
|
93
|
+
- lib/fluent/plugin/out_metricsense.rb
|
|
94
|
+
homepage: https://github.com/treasure-data/fluent-plugin-metricsense
|
|
95
|
+
licenses: []
|
|
96
|
+
post_install_message:
|
|
97
|
+
rdoc_options: []
|
|
98
|
+
require_paths:
|
|
99
|
+
- lib
|
|
100
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
101
|
+
none: false
|
|
102
|
+
requirements:
|
|
103
|
+
- - ! '>='
|
|
104
|
+
- !ruby/object:Gem::Version
|
|
105
|
+
version: '0'
|
|
106
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
107
|
+
none: false
|
|
108
|
+
requirements:
|
|
109
|
+
- - ! '>='
|
|
110
|
+
- !ruby/object:Gem::Version
|
|
111
|
+
version: '0'
|
|
112
|
+
requirements: []
|
|
113
|
+
rubyforge_project:
|
|
114
|
+
rubygems_version: 1.8.23
|
|
115
|
+
signing_key:
|
|
116
|
+
specification_version: 3
|
|
117
|
+
summary: MetricSense - application metrics aggregation plugin for Fluentd
|
|
118
|
+
test_files: []
|
|
119
|
+
has_rdoc: false
|