fluent-plugin-mongokpi 0.0.0 → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +130 -9
- data/fluent-plugin-mongokpi.gemspec +1 -1
- data/lib/fluent/plugin/out_mongokpi.rb +124 -144
- data/test/fluent/plugin/out_mongokpi_test.rb +2 -1
- metadata +2 -2
data/README.md
CHANGED
|
@@ -1,24 +1,144 @@
|
|
|
1
1
|
# Fluent::Plugin::Mongokpi
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## Description
|
|
4
|
+
Fluent BufferedOutput plugin: counting chunk, inserting counts to MongoDB
|
|
4
5
|
|
|
5
6
|
## Installation
|
|
7
|
+
$ gem install fluent-plugin-mongokpi
|
|
6
8
|
|
|
7
|
-
|
|
9
|
+
## Configuration
|
|
10
|
+
config | default | info
|
|
11
|
+
-------------------|-----------------|-----
|
|
12
|
+
address | localhost:27017 | address of mongodb (for replicaset, plz set like localhost:27017,localhost:27018,localhost:27019)
|
|
13
|
+
db | kpidb | db name of mongodb
|
|
14
|
+
collection | kpiColyyyymmdd | collection name of mongodb (yyyymmdd will be replaced based on time)
|
|
15
|
+
write_concern | | mongodb connection option
|
|
16
|
+
name | | mongodb connection option
|
|
17
|
+
read | | mongodb connection option
|
|
18
|
+
refresh_mode | | mongodb connection option
|
|
19
|
+
refresh_interval | | mongodb connection option
|
|
20
|
+
kpi_type | | use 'access' for activating access count
|
|
21
|
+
kpi_unit | min | time unit of count [min,hour,day]
|
|
22
|
+
time_key | | this time_key field or time of record is used
|
|
23
|
+
time_format | | (with time_key) the value of time_key will be parsed with this format
|
|
24
|
+
count_key | | used like group by (for multiple keys, plz set like site,host,instance)
|
|
25
|
+
count_name | count | the count result will be set to this field
|
|
26
|
+
response_threshold | 1000000 | (with kpi_type = access) used as threshold of response time
|
|
27
|
+
f_code | code | (with kpi_type = access) field name of response code in the log records
|
|
28
|
+
f_response_time | response_time | (with kpi_type = access) field name of response time in the log records
|
|
8
29
|
|
|
9
|
-
|
|
30
|
+
## MongoDB document
|
|
31
|
+
field | notification | info
|
|
32
|
+
----------------|--------------|-----
|
|
33
|
+
_id | | time + value of ${count_key}
|
|
34
|
+
yyyymmdd | |
|
|
35
|
+
hh | | (with kpi_unit = min or hour)
|
|
36
|
+
mm | | (with kpi_unit = min)
|
|
37
|
+
${count_key} | |
|
|
38
|
+
${count_name} | | count
|
|
39
|
+
count1xx | | (with kpi_type = access) count of ${f_code} = 1XX
|
|
40
|
+
count2xx | | (with kpi_type = access) count of ${f_code} = 2XX
|
|
41
|
+
count3xx | | (with kpi_type = access) count of ${f_code} = 3XX
|
|
42
|
+
count4xx | | (with kpi_type = access) count of ${f_code} = 4XX
|
|
43
|
+
count5xx | | (with kpi_type = access) count of ${f_code} = 5XX
|
|
44
|
+
responseTimeAve | `m(_ _)m` | (with kpi_type = access) average of ${f_response_time}
|
|
45
|
+
responseTimeMax | `m(_ _)m` | (with kpi_type = access) maximum of ${f_response_time}
|
|
46
|
+
responseTimeMin | `m(_ _)m` | (with kpi_type = access) minimum of ${f_response_time}
|
|
47
|
+
responseTimeSum | | (with kpi_type = access) sum of ${f_response_time}
|
|
48
|
+
countOver | | (with kpi_type = access) count of the access: ${f_response_time} >= ${response_threshold}
|
|
49
|
+
okRatio | `m(_ _)m` | (with kpi_type = access) (count - countOver) / count
|
|
50
|
+
counter[60] | | (with kpi_type = access & kpy_unit = min) qps for each second
|
|
51
|
+
qpsAve | `m(_ _)m` | (with kpi_type = access & kpy_unit = min) average of qps
|
|
52
|
+
qpsMax | `m(_ _)m` | (with kpi_type = access & kpy_unit = min) maximum of qps
|
|
53
|
+
qpsMin | `m(_ _)m` | (with kpi_type = access & kpy_unit = min) minimum of qps
|
|
10
54
|
|
|
11
|
-
|
|
55
|
+
`m(_ _)m` : these parameters are updated by non-atmic queries, then not reliable in multi processing
|
|
12
56
|
|
|
13
|
-
|
|
57
|
+
## Samples
|
|
58
|
+
### sample: access log (simple)
|
|
59
|
+
config
|
|
14
60
|
|
|
15
|
-
|
|
61
|
+
<match ***>
|
|
62
|
+
type mongokpi
|
|
63
|
+
kpi_type access
|
|
64
|
+
kpi_unit hourly
|
|
65
|
+
count_key site
|
|
66
|
+
time_key request_time
|
|
67
|
+
time_format %d/%b/%Y:%H:%M:%S %Z
|
|
68
|
+
</match>
|
|
16
69
|
|
|
17
|
-
|
|
70
|
+
log
|
|
71
|
+
|
|
72
|
+
{"request_time":"25/Nov/2013:20:03:02 +0700","code":"100","response_time":"1000","site":"git"}
|
|
73
|
+
{"request_time":"25/Nov/2013:20:03:03 +0700","code":"200","response_time":"2000","site":"git"}
|
|
74
|
+
{"request_time":"25/Nov/2013:20:03:03 +0700","code":"300","response_time":"3000","site":"git"}
|
|
75
|
+
{"request_time":"25/Nov/2013:20:03:03 +0700","code":"404","response_time":"4000","site":"blog"}
|
|
76
|
+
{"request_time":"25/Nov/2013:20:03:03 +0700","code":"500","response_time":"5000","site":"blog"}
|
|
77
|
+
|
|
78
|
+
MongoDB
|
|
79
|
+
|
|
80
|
+
db.kpiCol20131125.find()
|
|
81
|
+
{ "_id" : "git201311252003", "count" : 3, "count1xx" : 1, "count2xx" : 1, "count3xx" : 1, "count4xx" : 0, "count5xx" : 0, "countOver" : 0,
|
|
82
|
+
"hh" : "20", "okRatio" : 1, "responseTimeAve" : 2000, "responseTimeMax" : 3000, "responseTimeMin" : 1000, "responseTimeSum" : 6000, "site" : "git", "yyyymmdd" : "20131125"}
|
|
83
|
+
{ "_id" : "blog201311252003", "count" : 2, "count1xx" : 0, "count2xx" : 0, "count3xx" : 0, "count4xx" : 4, "count5xx" : 1, "countOver" : 0,
|
|
84
|
+
"hh" : "20", "okRatio" : 1, "responseTimeAve" : 4500, "responseTimeMax" : 5000, "responseTimeMin" : 4000, "responseTimeSum" : 9000, "site" : "blog", "yyyymmdd" : "20131125"}
|
|
85
|
+
{ "_id" : "total201311252003", "count" : 5, "count1xx" : 0, "count2xx" : 0, "count3xx" : 0, "count4xx" : 0, "count5xx" : 0, "countOver" : 0,
|
|
86
|
+
"hh" : "20", "okRatio" : 1, "responseTimeAve" : 3000, "responseTimeMax" : 5000, "responseTimeMin" : 1000, "responseTimeSum" : 15000, "site" : "total", "yyyymmdd" : "20131125" }
|
|
87
|
+
|
|
88
|
+
### sample: access log (detail)
|
|
89
|
+
config
|
|
90
|
+
|
|
91
|
+
<match ***>
|
|
92
|
+
type mongokpi
|
|
93
|
+
kpi_type access
|
|
94
|
+
count_key site
|
|
95
|
+
time_key request_time
|
|
96
|
+
time_format %d/%b/%Y:%H:%M:%S %Z
|
|
97
|
+
</match>
|
|
98
|
+
|
|
99
|
+
log
|
|
18
100
|
|
|
19
|
-
|
|
101
|
+
{"request_time":"25/Nov/2013:20:03:02 +0700","code":"100","response_time":"1000","site":"git"}
|
|
102
|
+
{"request_time":"25/Nov/2013:20:03:03 +0700","code":"200","response_time":"2000","site":"git"}
|
|
103
|
+
{"request_time":"25/Nov/2013:20:03:03 +0700","code":"300","response_time":"3000","site":"git"}
|
|
104
|
+
{"request_time":"25/Nov/2013:20:03:03 +0700","code":"404","response_time":"4000","site":"blog"}
|
|
105
|
+
{"request_time":"25/Nov/2013:20:03:03 +0700","code":"500","response_time":"5000","site":"blog"}
|
|
20
106
|
|
|
21
|
-
|
|
107
|
+
MongoDB
|
|
108
|
+
|
|
109
|
+
db.kpiCol20131125.find()
|
|
110
|
+
{ "_id" : "git201311252003", "count" : 3, "count1xx" : 1, "count2xx" : 1, "count3xx" : 1, "count4xx" : 0, "count5xx" : 0, "countOver" : 0,
|
|
111
|
+
"counter" : [ 0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
|
|
112
|
+
"hh" : "20", "mm" : "03", "okRatio" : 1, "qpsAve" : 0, "qpsMax" : 2, "qpsMin" : 0, "responseTimeAve" : 2000, "responseTimeMax" : 3000, "responseTimeMin" : 1000, "responseTimeSum" : 6000, "site" : "git", "yyyymmdd" : "20131125"}
|
|
113
|
+
{ "_id" : "blog201311252003", "count" : 2, "count1xx" : 0, "count2xx" : 0, "count3xx" : 0, "count4xx" : 4, "count5xx" : 1, "countOver" : 0,
|
|
114
|
+
"counter" : [ 0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
|
|
115
|
+
"hh" : "20", "mm" : "03", "okRatio" : 1, "qpsAve" : 0, "qpsMax" : 2, "qpsMin" : 0, "responseTimeAve" : 4500, "responseTimeMax" : 5000, "responseTimeMin" : 4000, "responseTimeSum" : 9000, "site" : "blog", "yyyymmdd" : "20131125"}
|
|
116
|
+
{ "_id" : "total201311252003", "count" : 5, "count1xx" : 0, "count2xx" : 0, "count3xx" : 0, "count4xx" : 0, "count5xx" : 0, "countOver" : 0,
|
|
117
|
+
"counter" : [ 0,0,1,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
|
|
118
|
+
"hh" : "20", "mm" : "03", "okRatio" : 1, "qpsAve" : 0, "qpsMax" : 4, "qpsMin" : 0, "responseTimeAve" : 3000, "responseTimeMax" : 5000, "responseTimeMin" : 1000, "responseTimeSum" : 15000, "site" : "total", "yyyymmdd" : "20131125" }
|
|
119
|
+
|
|
120
|
+
### sample: app log
|
|
121
|
+
config
|
|
122
|
+
|
|
123
|
+
<match ***>
|
|
124
|
+
type mongokpi
|
|
125
|
+
count_key site
|
|
126
|
+
</match>
|
|
127
|
+
|
|
128
|
+
log
|
|
129
|
+
|
|
130
|
+
{"gerogero":"Let's get Rocking!","site":"git"}
|
|
131
|
+
{"gerogero":"Let's get Rocking!","site":"git"}
|
|
132
|
+
{"gerogero":"Let's get Rocking!","site":"git"}
|
|
133
|
+
{"gerogero":"Let's get Rocking!","site":"blog"}
|
|
134
|
+
{"gerogero":"Let's get Rocking!","site":"blog"}
|
|
135
|
+
|
|
136
|
+
MongoDB
|
|
137
|
+
|
|
138
|
+
db.kpiCol20131125.find()
|
|
139
|
+
{ "_id" : "git201311252248", "count" : 3, "hh" : "22", "mm" : "48", "site" : "git", "yyyymmdd" : "20131125" }
|
|
140
|
+
{ "_id" : "blog201311252248", "count" : 2, "hh" : "22", "mm" : "48", "site" : "blog", "yyyymmdd" : "20131125" }
|
|
141
|
+
{ "_id" : "total201311252248","count" : 5, "hh" : "22", "mm" : "48", "site" : "total", "yyyymmdd" : "20131125" }
|
|
22
142
|
|
|
23
143
|
## Contributing
|
|
24
144
|
|
|
@@ -28,3 +148,4 @@ Or install it yourself as:
|
|
|
28
148
|
4. Push to the branch (`git push origin my-new-feature`)
|
|
29
149
|
5. Create new Pull Request
|
|
30
150
|
|
|
151
|
+
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "fluent-plugin-mongokpi"
|
|
7
|
-
spec.version = "0.0.
|
|
7
|
+
spec.version = "0.0.1"
|
|
8
8
|
spec.authors = ["Hatayama Hideharu"]
|
|
9
9
|
spec.email = ["h.hiddy@gmail.com"]
|
|
10
10
|
spec.description = %q{Fluent BufferedOutput plugin: counting chunk, inserting counts to make kpi count on MongoDB}
|
|
@@ -4,6 +4,13 @@ module Fluent
|
|
|
4
4
|
# which is used in the configuration file.
|
|
5
5
|
Plugin.register_output('mongokpi', self)
|
|
6
6
|
|
|
7
|
+
UNIT_DAYS = ['d', 'day', 'dayly']
|
|
8
|
+
UNIT_HOURS = ['h', 'hour', 'hourly']
|
|
9
|
+
UNIT_MINUTES = ['m', 'min', 'minute']
|
|
10
|
+
DAY = 'day'
|
|
11
|
+
HOUR = 'hour'
|
|
12
|
+
MIN = 'min'
|
|
13
|
+
|
|
7
14
|
# Mongo client settings
|
|
8
15
|
config_param :address, :string, :default => 'localhost:27017'
|
|
9
16
|
config_param :db, :string, :default => 'kpidb'
|
|
@@ -17,18 +24,18 @@ module Fluent
|
|
|
17
24
|
# collection options
|
|
18
25
|
config_param :capped_size, :integer, :default => 0
|
|
19
26
|
config_param :capped_max, :integer, :default => 0
|
|
20
|
-
#
|
|
27
|
+
# count options
|
|
28
|
+
config_param :kpi_type, :string, :default => nil # use 'access' for activating access count.
|
|
29
|
+
config_param :kpi_unit, :string, :default => 'min'
|
|
21
30
|
config_param :time_key, :string, :default => nil
|
|
22
31
|
config_param :time_format, :string, :default => nil
|
|
23
32
|
config_param :count_key, :string, :default => 'none'
|
|
24
33
|
config_param :count_name, :string, :default => 'count'
|
|
25
34
|
# access count options
|
|
26
|
-
config_param :
|
|
27
|
-
config_param :
|
|
28
|
-
config_param :
|
|
29
|
-
config_param :f_response_time, :string, :default => 'response_time'
|
|
35
|
+
config_param :response_threshold, :integer, :default => 1000000
|
|
36
|
+
config_param :f_code, :string, :default => 'code'
|
|
37
|
+
config_param :f_response_time, :string, :default => 'response_time'
|
|
30
38
|
|
|
31
|
-
|
|
32
39
|
attr_reader :collections_opts, :connection_opts
|
|
33
40
|
|
|
34
41
|
def initialize
|
|
@@ -43,6 +50,7 @@ module Fluent
|
|
|
43
50
|
# If the configuration is invalid, raise Fluent::ConfigError.
|
|
44
51
|
def configure(conf)
|
|
45
52
|
super
|
|
53
|
+
@kpi_unit = get_kpi_unit(@kpi_unit)
|
|
46
54
|
@connection_opts = {}
|
|
47
55
|
@connection_opts[:w] = @write_concern unless @write_concern.nil?
|
|
48
56
|
@connection_opts[:name] = @name unless @name.nil?
|
|
@@ -52,12 +60,20 @@ module Fluent
|
|
|
52
60
|
@collections_opts = {}
|
|
53
61
|
if @capped_size > 0
|
|
54
62
|
@collections_opts[:capped] = true
|
|
55
|
-
@collections_opts[:size] =
|
|
56
|
-
@collections_opts[:max] =
|
|
63
|
+
@collections_opts[:size] = @capped_size
|
|
64
|
+
@collections_opts[:max] = @capped_max if @capped_max
|
|
57
65
|
else
|
|
58
66
|
@collections_opts[:capped] = false
|
|
59
67
|
end
|
|
60
68
|
end
|
|
69
|
+
|
|
70
|
+
def get_kpi_unit(kpi_unit)
|
|
71
|
+
case kpi_unit
|
|
72
|
+
when *UNIT_DAYS then return DAY
|
|
73
|
+
when *UNIT_HOURS then return HOUR
|
|
74
|
+
else return MIN
|
|
75
|
+
end
|
|
76
|
+
end
|
|
61
77
|
|
|
62
78
|
# This method is called when starting.
|
|
63
79
|
# Open sockets or files here.
|
|
@@ -99,8 +115,8 @@ module Fluent
|
|
|
99
115
|
# }
|
|
100
116
|
#end
|
|
101
117
|
def write(chunk)
|
|
102
|
-
doc_hash = get_insert_doc_hash(@kpi_type,
|
|
103
|
-
insert(@kpi_type, @collection, @count_key, @count_name, doc_hash)
|
|
118
|
+
doc_hash = get_insert_doc_hash(@kpi_type, @kpi_unit, @time_key, @time_format, @count_key, @count_name, chunk)
|
|
119
|
+
insert(@kpi_type, @kpi_unit, @collection, @count_key, @count_name, doc_hash)
|
|
104
120
|
end
|
|
105
121
|
|
|
106
122
|
def get_client(address, connection_opts)
|
|
@@ -130,14 +146,14 @@ module Fluent
|
|
|
130
146
|
end
|
|
131
147
|
|
|
132
148
|
def convert_collection_name(collection_name, yyyymmdd)
|
|
133
|
-
return collection_name.sub('yyyymmdd', yyyymmdd)
|
|
149
|
+
return collection_name.sub('yyyymmdd', yyyymmdd.to_s)
|
|
134
150
|
end
|
|
135
151
|
|
|
136
152
|
def get_collection_from_db(client, db_name, collection_name, collections_opts)
|
|
137
153
|
return client.db(db_name).collection(collection_name, @collections_opts)
|
|
138
154
|
end
|
|
139
155
|
|
|
140
|
-
def get_insert_doc_hash(kpi_type,
|
|
156
|
+
def get_insert_doc_hash(kpi_type, kpi_unit, time_key, time_format, count_key, count_name, chunk)
|
|
141
157
|
hash_counter = {}
|
|
142
158
|
chunk.msgpack_each { |tag, time, record|
|
|
143
159
|
$log.debug record
|
|
@@ -148,26 +164,40 @@ module Fluent
|
|
|
148
164
|
if 'none' != count_key
|
|
149
165
|
count_key_value = ''
|
|
150
166
|
count_key.split(',').each { |x| count_key_value += record[x].to_s }
|
|
151
|
-
key_str = count_key_value + tmp_time
|
|
167
|
+
key_str = count_key_value + get_time_key_value(kpi_unit, tmp_time)
|
|
152
168
|
doc = hash_counter.key?(key_str) ? hash_counter[key_str]
|
|
153
|
-
: get_doc(kpi_type, count_key, count_key_value, count_name, tmp_time)
|
|
154
|
-
hash_counter[key_str] = count_up(kpi_type, doc, record, count_name, tmp_time)
|
|
169
|
+
: get_doc(kpi_type, kpi_unit, count_key, count_key_value, count_name, tmp_time)
|
|
170
|
+
hash_counter[key_str] = count_up(kpi_type, kpi_unit, doc, record, count_name, tmp_time)
|
|
155
171
|
end
|
|
156
172
|
# total
|
|
157
|
-
total_key_str = 'total' + tmp_time
|
|
173
|
+
total_key_str = 'total' + get_time_key_value(kpi_unit, tmp_time)
|
|
158
174
|
total = hash_counter.key?(total_key_str) ? hash_counter[total_key_str]
|
|
159
|
-
: get_doc(kpi_type, count_key, 'total', count_name, tmp_time)
|
|
160
|
-
hash_counter[total_key_str] = count_up(kpi_type, total, record, count_name, tmp_time)
|
|
175
|
+
: get_doc(kpi_type, kpi_unit, count_key, 'total', count_name, tmp_time)
|
|
176
|
+
hash_counter[total_key_str] = count_up(kpi_type, kpi_unit, total, record, count_name, tmp_time)
|
|
161
177
|
}
|
|
162
178
|
return hash_counter
|
|
163
179
|
end
|
|
164
180
|
|
|
165
|
-
def
|
|
181
|
+
def get_time_key_value(kpi_unit, time)
|
|
182
|
+
case kpi_unit
|
|
183
|
+
when DAY then return time.strftime('%Y%m%d')
|
|
184
|
+
when HOUR then return time.strftime('%Y%m%d%H')
|
|
185
|
+
else return time.strftime('%Y%m%d%H%M')
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def get_doc(kpi_type, kpi_unit, count_key, count_key_value, count_name, time)
|
|
166
190
|
doc = {}
|
|
167
191
|
doc[count_key] = count_key_value
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
192
|
+
time_field_hash = {}
|
|
193
|
+
time_field_hash['yyyymmdd'] = time.strftime('%Y%m%d').to_i
|
|
194
|
+
if HOUR == kpi_unit
|
|
195
|
+
time_field_hash['hh'] = time.strftime('%H').to_i
|
|
196
|
+
elsif MIN == kpi_unit
|
|
197
|
+
time_field_hash['hh'] = time.strftime('%H').to_i
|
|
198
|
+
time_field_hash['mm'] = time.strftime('%M').to_i
|
|
199
|
+
end
|
|
200
|
+
doc['time_field_hash'] = time_field_hash
|
|
171
201
|
doc[count_name] = 0
|
|
172
202
|
if 'access' == kpi_type
|
|
173
203
|
doc['countOver'] = 0
|
|
@@ -178,22 +208,24 @@ module Fluent
|
|
|
178
208
|
doc['count5xx'] = 0
|
|
179
209
|
doc['responseTimeAve'] = 0
|
|
180
210
|
doc['responseTimeMax'] = 0
|
|
181
|
-
doc['responseTimeMin'] = 100000000
|
|
211
|
+
doc['responseTimeMin'] = 100000000
|
|
182
212
|
doc['responseTimeSum'] = 0
|
|
183
|
-
doc['
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
213
|
+
doc['okRatio'] = 0.0000
|
|
214
|
+
if MIN == kpi_unit
|
|
215
|
+
doc['qpsAve'] = 0
|
|
216
|
+
doc['qpsMax'] = 0
|
|
217
|
+
doc['qpsMin'] = 100000000
|
|
218
|
+
doc['counter'] = Array.new(60, 0)
|
|
219
|
+
end
|
|
188
220
|
end
|
|
189
221
|
return doc
|
|
190
222
|
end
|
|
191
223
|
|
|
192
|
-
def count_up(kpi_type, doc, record, count_name, time)
|
|
224
|
+
def count_up(kpi_type, kpi_unit, doc, record, count_name, time)
|
|
193
225
|
doc[count_name] += 1
|
|
194
226
|
if 'access' == kpi_type
|
|
195
227
|
response_time = record.key?(@f_response_time) ? record[@f_response_time].to_i : 0
|
|
196
|
-
if response_time
|
|
228
|
+
if response_time >= @response_threshold
|
|
197
229
|
doc['countOver'] += 1
|
|
198
230
|
end
|
|
199
231
|
case record[@f_code].to_i / 100
|
|
@@ -210,140 +242,88 @@ module Fluent
|
|
|
210
242
|
doc['responseTimeMin'] = response_time
|
|
211
243
|
end
|
|
212
244
|
doc['responseTimeSum'] += response_time
|
|
213
|
-
doc['counter'][time.strftime('%S').to_i] += 1
|
|
245
|
+
doc['counter'][time.strftime('%S').to_i] += 1 if MIN == kpi_unit
|
|
214
246
|
end
|
|
215
247
|
return doc
|
|
216
248
|
end
|
|
217
249
|
|
|
218
|
-
|
|
250
|
+
# 2.5 or less
|
|
251
|
+
# http://stackoverflow.com/questions/8508663/calculate-max-value-in-an-atomic-findandmodify-operation
|
|
252
|
+
# TODO improve for Mongo 2.6
|
|
253
|
+
# $min, $max field update operators
|
|
254
|
+
# https://jira.mongodb.org/browse/SERVER-1534
|
|
255
|
+
# https://jira.mongodb.org/browse/DOCS-2012
|
|
256
|
+
def insert(kpi_type, kpi_unit, collection_name, count_key, count_name, doc_hash)
|
|
219
257
|
begin
|
|
220
258
|
doc_hash.each { |key, doc|
|
|
221
|
-
$log.debug doc
|
|
222
|
-
collection = get_collection(collection_name, doc['yyyymmdd'])
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
# $min, $max field update operators
|
|
227
|
-
# https://jira.mongodb.org/browse/SERVER-1534
|
|
228
|
-
# https://jira.mongodb.org/browse/DOCS-2012
|
|
259
|
+
$log.debug "key: #{key}, doc: #{doc}"
|
|
260
|
+
collection = get_collection(collection_name, doc['time_field_hash']['yyyymmdd'])
|
|
261
|
+
select_hash = doc['time_field_hash']
|
|
262
|
+
select_hash['_id'] = key
|
|
263
|
+
select_hash[count_key] = doc[count_key]
|
|
229
264
|
if 'access' == kpi_type
|
|
265
|
+
# initialize the target doc
|
|
266
|
+
# Without this, "0" in "counter.0' can be regarded as an child element, not as an array element.
|
|
267
|
+
if MIN == kpi_unit
|
|
268
|
+
collection.update(
|
|
269
|
+
select_hash,
|
|
270
|
+
{'$setOnInsert' => {'counter' => Array.new(60, 0)}},
|
|
271
|
+
{:upsert => true}
|
|
272
|
+
)
|
|
273
|
+
end
|
|
274
|
+
# main update
|
|
275
|
+
increment_hash = {}
|
|
276
|
+
increment_hash[count_name] = doc[count_name]
|
|
277
|
+
increment_hash['countOver'] = doc['countOver']
|
|
278
|
+
increment_hash['count1xx'] = doc['count1xx']
|
|
279
|
+
increment_hash['count2xx'] = doc['count2xx']
|
|
280
|
+
increment_hash['count3xx'] = doc['count3xx']
|
|
281
|
+
increment_hash['count4xx'] = doc['count4xx']
|
|
282
|
+
increment_hash['count5xx'] = doc['count5xx']
|
|
283
|
+
if MIN == kpi_unit
|
|
284
|
+
for sec in 0..59
|
|
285
|
+
increment_hash['counter.' + sec.to_s] = doc['counter'][sec]
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
increment_hash['responseTimeSum'] = doc['responseTimeSum']
|
|
230
289
|
collection.update(
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
{'$setOnInsert' => {'counter' => Array.new(60, 0)}},
|
|
234
|
-
{:upsert => true}
|
|
235
|
-
)
|
|
236
|
-
collection.update(
|
|
237
|
-
{'_id' => key, count_key => doc[count_key],
|
|
238
|
-
'yyyymmdd' => doc['yyyymmdd'], 'hh' => doc['hh'], 'mm' => doc['mm']},
|
|
239
|
-
{'$inc' => {
|
|
240
|
-
count_name => doc[count_name],
|
|
241
|
-
'countOver' => doc['countOver'],
|
|
242
|
-
'count1xx' => doc['count1xx'],
|
|
243
|
-
'count2xx' => doc['count2xx'],
|
|
244
|
-
'count3xx' => doc['count3xx'],
|
|
245
|
-
'count4xx' => doc['count4xx'],
|
|
246
|
-
'count5xx' => doc['count5xx'],
|
|
247
|
-
'responseTimeSum' => doc['responseTimeSum'],
|
|
248
|
-
'counter.0' => doc['counter'][0],
|
|
249
|
-
'counter.1' => doc['counter'][1],
|
|
250
|
-
'counter.2' => doc['counter'][2],
|
|
251
|
-
'counter.3' => doc['counter'][3],
|
|
252
|
-
'counter.4' => doc['counter'][4],
|
|
253
|
-
'counter.5' => doc['counter'][5],
|
|
254
|
-
'counter.6' => doc['counter'][6],
|
|
255
|
-
'counter.7' => doc['counter'][7],
|
|
256
|
-
'counter.8' => doc['counter'][8],
|
|
257
|
-
'counter.9' => doc['counter'][9],
|
|
258
|
-
'counter.10' => doc['counter'][10],
|
|
259
|
-
'counter.11' => doc['counter'][11],
|
|
260
|
-
'counter.12' => doc['counter'][12],
|
|
261
|
-
'counter.13' => doc['counter'][13],
|
|
262
|
-
'counter.14' => doc['counter'][14],
|
|
263
|
-
'counter.15' => doc['counter'][15],
|
|
264
|
-
'counter.16' => doc['counter'][16],
|
|
265
|
-
'counter.17' => doc['counter'][17],
|
|
266
|
-
'counter.18' => doc['counter'][18],
|
|
267
|
-
'counter.19' => doc['counter'][19],
|
|
268
|
-
'counter.20' => doc['counter'][20],
|
|
269
|
-
'counter.21' => doc['counter'][21],
|
|
270
|
-
'counter.22' => doc['counter'][22],
|
|
271
|
-
'counter.23' => doc['counter'][23],
|
|
272
|
-
'counter.24' => doc['counter'][24],
|
|
273
|
-
'counter.25' => doc['counter'][25],
|
|
274
|
-
'counter.26' => doc['counter'][26],
|
|
275
|
-
'counter.27' => doc['counter'][27],
|
|
276
|
-
'counter.28' => doc['counter'][28],
|
|
277
|
-
'counter.29' => doc['counter'][29],
|
|
278
|
-
'counter.30' => doc['counter'][30],
|
|
279
|
-
'counter.31' => doc['counter'][31],
|
|
280
|
-
'counter.32' => doc['counter'][32],
|
|
281
|
-
'counter.33' => doc['counter'][33],
|
|
282
|
-
'counter.34' => doc['counter'][34],
|
|
283
|
-
'counter.35' => doc['counter'][35],
|
|
284
|
-
'counter.36' => doc['counter'][36],
|
|
285
|
-
'counter.37' => doc['counter'][37],
|
|
286
|
-
'counter.38' => doc['counter'][38],
|
|
287
|
-
'counter.39' => doc['counter'][39],
|
|
288
|
-
'counter.40' => doc['counter'][40],
|
|
289
|
-
'counter.41' => doc['counter'][41],
|
|
290
|
-
'counter.42' => doc['counter'][42],
|
|
291
|
-
'counter.43' => doc['counter'][43],
|
|
292
|
-
'counter.44' => doc['counter'][44],
|
|
293
|
-
'counter.45' => doc['counter'][45],
|
|
294
|
-
'counter.46' => doc['counter'][46],
|
|
295
|
-
'counter.47' => doc['counter'][47],
|
|
296
|
-
'counter.48' => doc['counter'][48],
|
|
297
|
-
'counter.49' => doc['counter'][49],
|
|
298
|
-
'counter.50' => doc['counter'][50],
|
|
299
|
-
'counter.51' => doc['counter'][51],
|
|
300
|
-
'counter.52' => doc['counter'][52],
|
|
301
|
-
'counter.53' => doc['counter'][53],
|
|
302
|
-
'counter.54' => doc['counter'][54],
|
|
303
|
-
'counter.55' => doc['counter'][55],
|
|
304
|
-
'counter.56' => doc['counter'][56],
|
|
305
|
-
'counter.57' => doc['counter'][57],
|
|
306
|
-
'counter.58' => doc['counter'][58],
|
|
307
|
-
'counter.59' => doc['counter'][59]
|
|
308
|
-
}},
|
|
290
|
+
select_hash,
|
|
291
|
+
{'$inc' => increment_hash},
|
|
309
292
|
{:upsert => true}
|
|
310
293
|
)
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
294
|
+
# add supplemental fields using existing data
|
|
295
|
+
# NOTICE: this operation is not atmic, then the field value can be wrong in distributed processing.
|
|
296
|
+
updated_result = collection.find({'_id' => key})
|
|
297
|
+
if updated_result.nil?
|
|
298
|
+
$log.info "there is no updated result for the key: #{key}" if updated_result.nil?
|
|
299
|
+
continue
|
|
300
|
+
end
|
|
301
|
+
updated_doc = updated_result.to_a[0]
|
|
302
|
+
set_hash = {}
|
|
303
|
+
set_hash['responseTimeAve'] = updated_doc['responseTimeSum'] / doc[count_name]
|
|
316
304
|
if !updated_doc['responseTimeMax'].nil? && updated_doc['responseTimeMax'] > doc['responseTimeMax']
|
|
317
|
-
|
|
305
|
+
set_hash['responseTimeMax'] = updated_doc['responseTimeMax']
|
|
318
306
|
else
|
|
319
|
-
|
|
307
|
+
set_hash['responseTimeMax'] = doc['responseTimeMax']
|
|
320
308
|
end
|
|
321
309
|
if !updated_doc['responseTimeMin'].nil? && updated_doc['responseTimeMin'] < doc['responseTimeMin']
|
|
322
|
-
|
|
310
|
+
set_hash['responseTimeMin'] = updated_doc['responseTimeMin']
|
|
323
311
|
else
|
|
324
|
-
|
|
312
|
+
set_hash['responseTimeMin'] = doc['responseTimeMin']
|
|
313
|
+
end
|
|
314
|
+
set_hash['okRatio'] = ((updated_doc[count_name] - updated_doc['countOver']).to_f / updated_doc[count_name]).round(4)
|
|
315
|
+
if MIN == kpi_unit
|
|
316
|
+
set_hash['qpsAve'] = (updated_doc['counter'].inject(0.0){|r,i| r+=i } / updated_doc['counter'].size).round
|
|
317
|
+
set_hash['qpsMax'] = updated_doc['counter'].max
|
|
318
|
+
set_hash['qpsMin'] = updated_doc['counter'].min
|
|
325
319
|
end
|
|
326
|
-
qps_ave = (updated_doc['counter'].inject(0.0){|r,i| r+=i } / updated_doc['counter'].size).round
|
|
327
|
-
qps_max = updated_doc['counter'].max
|
|
328
|
-
qps_min = updated_doc['counter'].min
|
|
329
|
-
ok_ratio = ((updated_doc[count_name] - updated_doc['countOver']).to_f / updated_doc[count_name]).round(4)
|
|
330
320
|
collection.update(
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
{ '$set' => {
|
|
334
|
-
'responseTimeAve' => response_time_ave,
|
|
335
|
-
'responseTimeMax' => response_time_max,
|
|
336
|
-
'responseTimeMin' => response_time_min,
|
|
337
|
-
'qpsAve' => qps_ave,
|
|
338
|
-
'qpsMax' => qps_max,
|
|
339
|
-
'qpsMin' => qps_min,
|
|
340
|
-
'okRatio' => ok_ratio
|
|
341
|
-
}}
|
|
321
|
+
select_hash,
|
|
322
|
+
{ '$set' => set_hash}
|
|
342
323
|
)
|
|
343
324
|
else
|
|
344
325
|
collection.update(
|
|
345
|
-
|
|
346
|
-
'yyyymmdd' => doc['yyyymmdd'], 'hh' => doc['hh'], 'mm' => doc['mm']},
|
|
326
|
+
select_hash,
|
|
347
327
|
{'$inc' => {count_name => doc[count_name]}},
|
|
348
328
|
{:upsert => true}
|
|
349
329
|
)
|
|
@@ -57,7 +57,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
|
57
57
|
|
|
58
58
|
def test_format
|
|
59
59
|
d = create_driver
|
|
60
|
-
time = Time.parse("2112-09-03 01:23:45 UTC")
|
|
60
|
+
time = Time.parse("2112-09-03 01:23:45 UTC")
|
|
61
61
|
d.emit({"gerogero" => "Let's get Rocking!", "site" => "yapoo"}, time)
|
|
62
62
|
d.emit({"gerogero" => "Let's get Rocking!", "site" => "geegero"}, time)
|
|
63
63
|
d.expect_format "\x93\xABdefault_tag\xCF\x00\x00\x00\x01\f[\xC8\xA1\x82\xA8gerogero\xB2Let's get Rocking!\xA4site\xA5yapoo\x93\xABdefault_tag\xCF\x00\x00\x00\x01\f[\xC8\xA1\x82\xA8gerogero\xB2Let's get Rocking!\xA4site\xA7geegero"
|
|
@@ -88,3 +88,4 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
|
88
88
|
end
|
|
89
89
|
|
|
90
90
|
|
|
91
|
+
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-mongokpi
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.1
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2013-11-
|
|
12
|
+
date: 2013-11-28 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: bundler
|