fluent-plugin-rtf-dash 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +4 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +14 -0
- data/README.md +14 -0
- data/Rakefile +10 -0
- data/examples/example.conf +156 -0
- data/examples/example2.conf +77 -0
- data/fluent-plugin-rtf-dash.gemspec +20 -0
- data/lib/fluent/plugin/out_rtf_dash.rb +121 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_rtf_dash.rb +207 -0
- metadata +86 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f0c1ac340bf42b98e66982ecf2f808995ad80480
|
4
|
+
data.tar.gz: fd8f15b8afa45dbc381ef55fab9b05046538516e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a3501229718d8c3943a45251da7055439662af2d82d046ea63d1f62fbdd01d2b75293c1d1cdc91ebcb1fc1e07b8a0cfcc9ac2bd4b3cb503d43f8a2fc7d64ffe1
|
7
|
+
data.tar.gz: 590a03cfb5a3d0edeece62749960ca32e3a303227bcdbc73fc644f644cef6c83318c956a81ac182ed0d3f3b2ca8426e780249ebde5f958f47cb3365df7c72c8d
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright (c) 2012- Kentaro Yoshida
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
14
|
+
|
data/README.md
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# fluent-plugin-rtf-dash
|
2
|
+
|
3
|
+
## Copyright
|
4
|
+
|
5
|
+
### reference: [fluent-plugin-rewrite-tag-filter](https://github.com/fluent/fluent-plugin-rewrite-tag-filter)
|
6
|
+
|
7
|
+
- Copyright : Copyright (c) 2012- Kentaro Yoshida (@yoshi_ken)
|
8
|
+
- License : Apache License, Version 2.0
|
9
|
+
|
10
|
+
### present package: fluent-plugin-rtf-dash
|
11
|
+
|
12
|
+
- Copyright : Copyright (c) 2013 Aiming Inc.
|
13
|
+
- License : Apache License, Version 2.0
|
14
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,156 @@
|
|
1
|
+
# It is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.
|
2
|
+
|
3
|
+
# appearing plugins:
|
4
|
+
# rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
|
5
|
+
# forest: http://rubygems.org/gems/fluent-plugin-forest
|
6
|
+
# datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
|
7
|
+
# growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
|
8
|
+
|
9
|
+
<source>
|
10
|
+
type tail
|
11
|
+
path /var/log/httpd/access_log
|
12
|
+
format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
|
13
|
+
time_format %d/%b/%Y:%H:%M:%S %z
|
14
|
+
tag td.apache.access
|
15
|
+
pos_file /var/log/td-agent/apache_access.pos
|
16
|
+
</source>
|
17
|
+
|
18
|
+
|
19
|
+
# Extract specified virtual domain
|
20
|
+
<match td.apache.access>
|
21
|
+
type copy
|
22
|
+
<store>
|
23
|
+
type rewrite_tag_filter
|
24
|
+
capitalize_regex_backreference yes
|
25
|
+
rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
|
26
|
+
</store>
|
27
|
+
<store>
|
28
|
+
type rewrite_tag_filter
|
29
|
+
capitalize_regex_backreference yes
|
30
|
+
rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
|
31
|
+
</store>
|
32
|
+
</match>
|
33
|
+
|
34
|
+
|
35
|
+
# Second level analyzing
|
36
|
+
<match sitepath.GoogleMaps>
|
37
|
+
<store>
|
38
|
+
type rewrite_tag_filter
|
39
|
+
rewriterule1 path ^/labs site.GoogleMaps.Labs
|
40
|
+
rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
|
41
|
+
rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
|
42
|
+
</store>
|
43
|
+
<store>
|
44
|
+
type rewrite_tag_filter
|
45
|
+
rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
|
46
|
+
rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
|
47
|
+
</store>
|
48
|
+
<store>
|
49
|
+
type rewrite_tag_filter
|
50
|
+
rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
|
51
|
+
rewriterule2 agent ^.* iPhone .+Googlebot-Mobile/.*$ site.GoogleMaps.agent_GooglebotSmartphone
|
52
|
+
rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
|
53
|
+
rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
|
54
|
+
rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
|
55
|
+
</store>
|
56
|
+
</match>
|
57
|
+
|
58
|
+
|
59
|
+
# Summarize
|
60
|
+
<match site.**>
|
61
|
+
type copy
|
62
|
+
<store>
|
63
|
+
type forest
|
64
|
+
subtype datacounter
|
65
|
+
<template>
|
66
|
+
unit minute
|
67
|
+
count_key response_time
|
68
|
+
outcast_unmatched false
|
69
|
+
aggregate all
|
70
|
+
tag gf.responsetime.__TAG__
|
71
|
+
pattern1 0-100msec ^\d{1,5}$
|
72
|
+
pattern2 100-300msec ^[1-2]\d{5}$
|
73
|
+
pattern3 300-600msec ^[3-5]\d{5}$
|
74
|
+
pattern4 600msec-1sec ^[6-9]\d{5}$
|
75
|
+
pattern5 1-2sec ^1\d{6}$
|
76
|
+
pattern6 2-10sec ^[2-9]\d{6}$
|
77
|
+
pattern7 10sec_over ^\d{8,}$
|
78
|
+
</template>
|
79
|
+
</store>
|
80
|
+
<store>
|
81
|
+
type forest
|
82
|
+
subtype datacounter
|
83
|
+
<template>
|
84
|
+
unit minute
|
85
|
+
outcast_unmatched false
|
86
|
+
aggregate all
|
87
|
+
tag gf.responsecode.__TAG__
|
88
|
+
count_key status
|
89
|
+
pattern1 200 ^200$
|
90
|
+
pattern2 2xx ^2\d\d$
|
91
|
+
pattern3 301 ^301$
|
92
|
+
pattern4 302 ^302$
|
93
|
+
pattern5 3xx ^3\d\d$
|
94
|
+
pattern6 403 ^403$
|
95
|
+
pattern7 404 ^404$
|
96
|
+
pattern8 410 ^410$
|
97
|
+
pattern9 4xx ^4\d\d$
|
98
|
+
pattern10 500 ^5\d\d$
|
99
|
+
</template>
|
100
|
+
</store>
|
101
|
+
<store>
|
102
|
+
type forest
|
103
|
+
subtype datacounter
|
104
|
+
<template>
|
105
|
+
unit minute
|
106
|
+
count_key agent
|
107
|
+
outcast_unmatched false
|
108
|
+
aggregate all
|
109
|
+
tag gf.useragent.__TAG__
|
110
|
+
pattern1 api HttpRequest
|
111
|
+
pattern2 robot (spider|bot|crawler|\+http\:)
|
112
|
+
pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
|
113
|
+
pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
|
114
|
+
pattern5 pc .+
|
115
|
+
</template>
|
116
|
+
</store>
|
117
|
+
</match>
|
118
|
+
|
119
|
+
|
120
|
+
# Graph
|
121
|
+
<match gf.responsetime.**>
|
122
|
+
type forest
|
123
|
+
subtype growthforecast
|
124
|
+
remove_prefix gf.responsetime.site
|
125
|
+
<template>
|
126
|
+
gfapi_url http://localhost:5125/api/
|
127
|
+
service __TAG__
|
128
|
+
section response_time
|
129
|
+
name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
|
130
|
+
</template>
|
131
|
+
</match>
|
132
|
+
|
133
|
+
<match gf.responsecode.**>
|
134
|
+
type forest
|
135
|
+
subtype growthforecast
|
136
|
+
remove_prefix gf.responsecode.site
|
137
|
+
<template>
|
138
|
+
gfapi_url http://localhost:5125/api/
|
139
|
+
service __TAG__
|
140
|
+
section response_code
|
141
|
+
name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
|
142
|
+
</template>
|
143
|
+
</match>
|
144
|
+
|
145
|
+
<match gf.useragent.**>
|
146
|
+
type forest
|
147
|
+
subtype growthforecast
|
148
|
+
remove_prefix gf.useragent.site
|
149
|
+
<template>
|
150
|
+
gfapi_url http://localhost:5125/api/
|
151
|
+
service __TAG__
|
152
|
+
section useragent
|
153
|
+
name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
|
154
|
+
</template>
|
155
|
+
</match>
|
156
|
+
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# It is a sample how to exclude specified patterns before analyze response_time for each virtual domain websites.
|
2
|
+
|
3
|
+
# appearing plugins:
|
4
|
+
# rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
|
5
|
+
# forest: http://rubygems.org/gems/fluent-plugin-forest
|
6
|
+
# datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
|
7
|
+
# growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
|
8
|
+
|
9
|
+
<source>
|
10
|
+
type tail
|
11
|
+
path /var/log/httpd/access_log
|
12
|
+
format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
|
13
|
+
time_format %d/%b/%Y:%H:%M:%S %z
|
14
|
+
tag td.apache.access
|
15
|
+
pos_file /var/log/td-agent/apache_access.pos
|
16
|
+
</source>
|
17
|
+
|
18
|
+
|
19
|
+
# Extract specified virtual domain
|
20
|
+
<match td.apache.access>
|
21
|
+
type rewrite_tag_filter
|
22
|
+
rewriterule1 domain ^maps\.google\.com$ filter.GoogleMap
|
23
|
+
</match>
|
24
|
+
|
25
|
+
|
26
|
+
# Filtering
|
27
|
+
<match filter.GoogleMap>
|
28
|
+
type rewrite_tag_filter
|
29
|
+
rewriterule1 path ^/(img|css|js|static|assets)/ clear
|
30
|
+
rewriterule2 status ^(?!200)$ clear
|
31
|
+
rewriterule3 method ^(?!GET)$ clear
|
32
|
+
rewriterule4 agent (spider|bot|crawler|\+http\:) clear
|
33
|
+
rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
|
34
|
+
rewriterule6 path .+ site.GoogleMap.front
|
35
|
+
</match>
|
36
|
+
|
37
|
+
|
38
|
+
# Summarize
|
39
|
+
<match site.**>
|
40
|
+
type forest
|
41
|
+
subtype datacounter
|
42
|
+
<template>
|
43
|
+
unit minute
|
44
|
+
count_key response_time
|
45
|
+
outcast_unmatched false
|
46
|
+
aggregate all
|
47
|
+
tag gf.responsetime.__TAG__
|
48
|
+
pattern1 0-100msec ^\d{1,5}$
|
49
|
+
pattern2 100-300msec ^[1-2]\d{5}$
|
50
|
+
pattern3 300-600msec ^[3-5]\d{5}$
|
51
|
+
pattern4 600msec-1sec ^[6-9]\d{5}$
|
52
|
+
pattern5 1-2sec ^1\d{6}$
|
53
|
+
pattern6 2-10sec ^[2-9]\d{6}$
|
54
|
+
pattern7 10sec_over ^\d{8,}$
|
55
|
+
</template>
|
56
|
+
</match>
|
57
|
+
|
58
|
+
|
59
|
+
# Graph
|
60
|
+
<match gf.responsetime.**>
|
61
|
+
type forest
|
62
|
+
subtype growthforecast
|
63
|
+
remove_prefix gf.responsetime.site
|
64
|
+
<template>
|
65
|
+
gfapi_url http://localhost:5125/api/
|
66
|
+
service __TAG__
|
67
|
+
section response_time
|
68
|
+
name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
|
69
|
+
</template>
|
70
|
+
</match>
|
71
|
+
|
72
|
+
|
73
|
+
# Clear tag
|
74
|
+
<match clear>
|
75
|
+
type null
|
76
|
+
</match>
|
77
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = "fluent-plugin-rtf-dash"
|
6
|
+
s.version = "0.0.1"
|
7
|
+
s.license = "Apache 2.0"
|
8
|
+
s.authors = ["Aiming Inc."]
|
9
|
+
s.email = ["info@aiming-inc.com"]
|
10
|
+
s.homepage = ""
|
11
|
+
s.summary = ""
|
12
|
+
|
13
|
+
s.files = `git ls-files`.split("\n")
|
14
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
15
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
16
|
+
s.require_paths = ["lib"]
|
17
|
+
|
18
|
+
s.add_development_dependency "rake"
|
19
|
+
s.add_runtime_dependency "fluentd"
|
20
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
# -*- coding:utf-8 -*-
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
class RTFDashOutput < Fluent::Output
|
5
|
+
Fluent::Plugin.register_output('rtf_dash', self)
|
6
|
+
|
7
|
+
config_param :capitalize_regex_backreference, :bool, :default => false
|
8
|
+
config_param :remove_tag_prefix, :string, :default => nil
|
9
|
+
config_param :use_of_first_match_tag_regexp, :string, :default => nil
|
10
|
+
config_param :hostname_command, :string, :default => 'hostname'
|
11
|
+
|
12
|
+
MATCH_OPERATOR_EXCLUDE = '!'
|
13
|
+
|
14
|
+
def configure(conf)
|
15
|
+
super
|
16
|
+
|
17
|
+
@rewriterules = []
|
18
|
+
rewriterule_names = []
|
19
|
+
@hostname = `#{@hostname_command}`.chomp
|
20
|
+
|
21
|
+
conf.keys.select{|k| k =~ /^rewriterule(\d+)$/}.sort_by{|i| i.sub('rewriterule', '').to_i}.each do |key|
|
22
|
+
rewritekey,regexp,rewritetag = parse_rewriterule(conf[key])
|
23
|
+
if regexp.nil? || rewritetag.nil?
|
24
|
+
raise Fluent::ConfigError, "failed to parse rewriterules at #{key} #{conf[key]}"
|
25
|
+
end
|
26
|
+
@rewriterules.push([rewritekey, /#{trim_regex_quote(regexp)}/, get_match_operator(regexp), rewritetag])
|
27
|
+
rewriterule_names.push(rewritekey + regexp)
|
28
|
+
$log.info "adding rewrite_tag_filter rule: #{key} #{@rewriterules.last}"
|
29
|
+
end
|
30
|
+
|
31
|
+
unless @rewriterules.length > 0
|
32
|
+
raise Fluent::ConfigError, "missing rewriterules"
|
33
|
+
end
|
34
|
+
|
35
|
+
unless @rewriterules.length == rewriterule_names.uniq.length
|
36
|
+
raise Fluent::ConfigError, "duplicated rewriterules found #{@rewriterules.inspect}"
|
37
|
+
end
|
38
|
+
|
39
|
+
unless @remove_tag_prefix.nil?
|
40
|
+
@remove_tag_prefix = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
|
41
|
+
end
|
42
|
+
|
43
|
+
unless @use_of_first_match_tag_regexp.nil?
|
44
|
+
@use_of_first_match_tag_regexp = Regexp.new(@use_of_first_match_tag_regexp)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def emit(tag, es, chain)
|
49
|
+
es.each do |time,record|
|
50
|
+
rewrited_tag = rewrite_tag(tag, record)
|
51
|
+
next if rewrited_tag.nil? || tag == rewrited_tag
|
52
|
+
Fluent::Engine.emit(rewrited_tag, time, record)
|
53
|
+
end
|
54
|
+
|
55
|
+
chain.next
|
56
|
+
end
|
57
|
+
|
58
|
+
def rewrite_tag(tag, record)
|
59
|
+
placeholder = get_placeholder(tag)
|
60
|
+
@rewriterules.each do |rewritekey, regexp, match_operator, rewritetag|
|
61
|
+
rewritevalue = record[rewritekey].to_s
|
62
|
+
next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
|
63
|
+
matched = regexp && regexp.match(rewritevalue)
|
64
|
+
case match_operator
|
65
|
+
when MATCH_OPERATOR_EXCLUDE
|
66
|
+
next if matched
|
67
|
+
else
|
68
|
+
next if !matched
|
69
|
+
backreference_table = get_backreference_table($~.captures)
|
70
|
+
rewritetag = rewritetag.gsub(/\$\d+/, backreference_table)
|
71
|
+
end
|
72
|
+
rewritetag = rewritetag.gsub(/(\${[a-z]+(\[[0-9]+\])?}|__[A-Z]+__)/, placeholder)
|
73
|
+
return rewritetag
|
74
|
+
end
|
75
|
+
return nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def parse_rewriterule(rule)
|
79
|
+
if rule.match(/^([^\s]+)\s+(.+?)\s+([^\s]+)$/)
|
80
|
+
return $~.captures
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def trim_regex_quote(regexp)
|
85
|
+
if regexp.start_with?('"') && regexp.end_with?('"')
|
86
|
+
$log.info "rewrite_tag_filter: [DEPRECATED] Use ^....$ pattern for partial word match instead of double-quote-delimiter. #{regexp}"
|
87
|
+
regexp = regexp[1..-2]
|
88
|
+
end
|
89
|
+
if regexp.start_with?(MATCH_OPERATOR_EXCLUDE)
|
90
|
+
regexp = regexp[1, regexp.length]
|
91
|
+
end
|
92
|
+
return regexp
|
93
|
+
end
|
94
|
+
|
95
|
+
def get_match_operator(regexp)
|
96
|
+
return MATCH_OPERATOR_EXCLUDE if regexp.start_with?(MATCH_OPERATOR_EXCLUDE)
|
97
|
+
return ''
|
98
|
+
end
|
99
|
+
|
100
|
+
def get_backreference_table(elements)
|
101
|
+
hash_table = Hash.new
|
102
|
+
elements.each.with_index(1) do |value, index|
|
103
|
+
hash_table["$#{index}"] = @capitalize_regex_backreference ? value.capitalize : value
|
104
|
+
end
|
105
|
+
return hash_table
|
106
|
+
end
|
107
|
+
|
108
|
+
def get_placeholder(tag)
|
109
|
+
tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
|
110
|
+
tag = tag.match(@use_of_first_match_tag_regexp)[1] if @use_of_first_match_tag_regexp
|
111
|
+
p tag
|
112
|
+
return {
|
113
|
+
'__HOSTNAME__' => @hostname,
|
114
|
+
'${hostname}' => @hostname,
|
115
|
+
'__TAG__' => tag,
|
116
|
+
'${tag}' => tag,
|
117
|
+
}
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_rtf_dash'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,207 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class RTFDashOutputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
rewriterule1 domain ^www\.google\.com$ site.Google
|
10
|
+
rewriterule2 domain ^news\.google\.com$ site.GoogleNews
|
11
|
+
rewriterule3 agent .* Mac OS X .* agent.MacOSX
|
12
|
+
rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
|
13
|
+
rewriterule5 domain ^(tagtest)\.google\.com$ site.${tag}.$1
|
14
|
+
]
|
15
|
+
|
16
|
+
# aggresive test
|
17
|
+
# indentation, comment, capitalize_regex_backreference, regex with space aside.
|
18
|
+
# [DEPLICATED] Use ^....$ pattern for partial word match instead of double-quote-delimiter.
|
19
|
+
CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION = %[
|
20
|
+
capitalize_regex_backreference yes
|
21
|
+
rewriterule1 domain ^www\.google\.com$ site.Google # some comment
|
22
|
+
rewriterule2 domain ^(news)\.(google)\.com$ site.$2$1
|
23
|
+
rewriterule3 agent ^.* Mac OS X .*$ agent.MacOSX
|
24
|
+
rewriterule4 agent "(Googlebot|CustomBot)-([a-zA-Z]+)" agent.$1-$2
|
25
|
+
]
|
26
|
+
|
27
|
+
# remove_tag_prefix test
|
28
|
+
CONFIG_REMOVE_TAG_PREFIX = %[
|
29
|
+
rewriterule1 domain ^www\.google\.com$ ${tag}
|
30
|
+
remove_tag_prefix input
|
31
|
+
]
|
32
|
+
|
33
|
+
# remove_tag_prefix test2
|
34
|
+
CONFIG_REMOVE_TAG_PREFIX_WITH_DOT = %[
|
35
|
+
rewriterule1 domain ^www\.google\.com$ ${tag}
|
36
|
+
remove_tag_prefix input.
|
37
|
+
]
|
38
|
+
|
39
|
+
# hostname placeholder test
|
40
|
+
CONFIG_SHORT_HOSTNAME = %[
|
41
|
+
rewriterule1 domain ^www\.google\.com$ ${hostname}
|
42
|
+
remove_tag_prefix input
|
43
|
+
hostname_command hostname -s
|
44
|
+
]
|
45
|
+
|
46
|
+
# '!' character (exclamation mark) to specify a non-matching pattern
|
47
|
+
CONFIG_NON_MATCHING = %[
|
48
|
+
rewriterule1 domain !^www\..+$ not_start_with_www
|
49
|
+
rewriterule2 domain ^www\..+$ start_with_www
|
50
|
+
]
|
51
|
+
|
52
|
+
# jump of index
|
53
|
+
CONFIG_JUMP_INDEX = %[
|
54
|
+
rewriterule10 domain ^www\.google\.com$ site.Google
|
55
|
+
rewriterule20 domain ^news\.google\.com$ site.GoogleNews
|
56
|
+
]
|
57
|
+
|
58
|
+
CONFIG_USE_OF_FIRST_MATCH_TAG_REGEXP = %[
|
59
|
+
use_of_first_match_tag_regexp [a-z_]+\.([a-z_]+)\.
|
60
|
+
rewriterule1 type ^[a-z_]+$ api.${tag}.warrior
|
61
|
+
]
|
62
|
+
|
63
|
+
def create_driver(conf=CONFIG,tag='test')
|
64
|
+
Fluent::Test::OutputTestDriver.new(Fluent::RTFDashOutput, tag).configure(conf)
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_configure
|
68
|
+
assert_raise(Fluent::ConfigError) {
|
69
|
+
d = create_driver('')
|
70
|
+
}
|
71
|
+
assert_raise(Fluent::ConfigError) {
|
72
|
+
d = create_driver('rewriterule1 foo')
|
73
|
+
}
|
74
|
+
assert_raise(Fluent::ConfigError) {
|
75
|
+
d = create_driver('rewriterule1 foo foo')
|
76
|
+
}
|
77
|
+
d = create_driver %[
|
78
|
+
rewriterule1 domain ^www.google.com$ site.Google
|
79
|
+
rewriterule2 domain ^news.google.com$ site.GoogleNews
|
80
|
+
]
|
81
|
+
puts d.instance.inspect
|
82
|
+
assert_equal 'domain ^www.google.com$ site.Google', d.instance.config['rewriterule1']
|
83
|
+
assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.config['rewriterule2']
|
84
|
+
end
|
85
|
+
|
86
|
+
def test_emit
|
87
|
+
d1 = create_driver(CONFIG, 'input.access')
|
88
|
+
d1.run do
|
89
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
90
|
+
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
91
|
+
d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
92
|
+
d1.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
93
|
+
d1.emit({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
94
|
+
d1.emit({'domain' => 'noop.example.com'}) # to be ignored
|
95
|
+
end
|
96
|
+
emits = d1.emits
|
97
|
+
assert_equal 5, emits.length
|
98
|
+
p emits[0]
|
99
|
+
assert_equal 'site.Google', emits[0][0] # tag
|
100
|
+
p emits[1]
|
101
|
+
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
102
|
+
assert_equal 'news.google.com', emits[1][2]['domain']
|
103
|
+
p emits[2]
|
104
|
+
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
105
|
+
p emits[3]
|
106
|
+
assert_equal 'agent.Googlebot-FooBar', emits[3][0] #tag
|
107
|
+
p emits[4]
|
108
|
+
assert_equal 'site.input.access.tagtest', emits[4][0] #tag
|
109
|
+
end
|
110
|
+
|
111
|
+
def test_emit2_indent_and_capitalize_option
|
112
|
+
d1 = create_driver(CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION, 'input.access')
|
113
|
+
d1.run do
|
114
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
115
|
+
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
116
|
+
d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
117
|
+
d1.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
118
|
+
end
|
119
|
+
emits = d1.emits
|
120
|
+
assert_equal 4, emits.length
|
121
|
+
p emits[0]
|
122
|
+
assert_equal 'site.Google', emits[0][0] # tag
|
123
|
+
p emits[1]
|
124
|
+
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
125
|
+
assert_equal 'news.google.com', emits[1][2]['domain']
|
126
|
+
p emits[2]
|
127
|
+
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
128
|
+
p emits[3]
|
129
|
+
assert_equal 'agent.Googlebot-Foobar', emits[3][0] #tag
|
130
|
+
end
|
131
|
+
|
132
|
+
def test_emit3_remove_tag_prefix
|
133
|
+
d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX, 'input.access')
|
134
|
+
d1.run do
|
135
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
136
|
+
end
|
137
|
+
emits = d1.emits
|
138
|
+
assert_equal 1, emits.length
|
139
|
+
p emits[0]
|
140
|
+
assert_equal 'access', emits[0][0] # tag
|
141
|
+
end
|
142
|
+
|
143
|
+
def test_emit4_remove_tag_prefix_with_dot
|
144
|
+
d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX_WITH_DOT, 'input.access')
|
145
|
+
d1.run do
|
146
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
147
|
+
end
|
148
|
+
emits = d1.emits
|
149
|
+
assert_equal 1, emits.length
|
150
|
+
p emits[0]
|
151
|
+
assert_equal 'access', emits[0][0] # tag
|
152
|
+
end
|
153
|
+
|
154
|
+
def test_emit5_short_hostname
|
155
|
+
d1 = create_driver(CONFIG_SHORT_HOSTNAME, 'input.access')
|
156
|
+
d1.run do
|
157
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
158
|
+
end
|
159
|
+
emits = d1.emits
|
160
|
+
assert_equal 1, emits.length
|
161
|
+
p emits[0]
|
162
|
+
assert_equal `hostname -s`.chomp, emits[0][0] # tag
|
163
|
+
end
|
164
|
+
|
165
|
+
def test_emit6_non_matching
|
166
|
+
d1 = create_driver(CONFIG_NON_MATCHING, 'input.access')
|
167
|
+
d1.run do
|
168
|
+
d1.emit({'domain' => 'www.google.com'})
|
169
|
+
d1.emit({'path' => '/'})
|
170
|
+
d1.emit({'domain' => 'maps.google.com'})
|
171
|
+
end
|
172
|
+
emits = d1.emits
|
173
|
+
assert_equal 3, emits.length
|
174
|
+
p emits[0]
|
175
|
+
assert_equal 'start_with_www', emits[0][0] # tag
|
176
|
+
p emits[1]
|
177
|
+
assert_equal 'not_start_with_www', emits[1][0] # tag
|
178
|
+
p emits[2]
|
179
|
+
assert_equal 'not_start_with_www', emits[2][0] # tag
|
180
|
+
end
|
181
|
+
|
182
|
+
def test_emit7_jump_index
|
183
|
+
d1 = create_driver(CONFIG_JUMP_INDEX, 'input.access')
|
184
|
+
d1.run do
|
185
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
186
|
+
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
187
|
+
end
|
188
|
+
emits = d1.emits
|
189
|
+
assert_equal 2, emits.length
|
190
|
+
p emits[0]
|
191
|
+
assert_equal 'site.Google', emits[0][0] # tag
|
192
|
+
p emits[1]
|
193
|
+
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
194
|
+
end
|
195
|
+
|
196
|
+
def test_emit8_first_match_tag
|
197
|
+
d1 = create_driver(CONFIG_USE_OF_FIRST_MATCH_TAG_REGEXP, 'hoge_game.production.api')
|
198
|
+
d1.run do
|
199
|
+
d1.emit({'user_id' => '1000', 'type' => 'warrior', 'name' => 'Richard Costner'})
|
200
|
+
end
|
201
|
+
emits = d1.emits
|
202
|
+
p emits[0]
|
203
|
+
assert_equal 1, emits.length
|
204
|
+
assert_equal 'api.production.warrior', emits[0][0] # tag
|
205
|
+
end
|
206
|
+
|
207
|
+
end
|
metadata
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-rtf-dash
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Aiming Inc.
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-12-05 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
description:
|
42
|
+
email:
|
43
|
+
- info@aiming-inc.com
|
44
|
+
executables: []
|
45
|
+
extensions: []
|
46
|
+
extra_rdoc_files: []
|
47
|
+
files:
|
48
|
+
- .gitignore
|
49
|
+
- .travis.yml
|
50
|
+
- Gemfile
|
51
|
+
- LICENSE.txt
|
52
|
+
- README.md
|
53
|
+
- Rakefile
|
54
|
+
- examples/example.conf
|
55
|
+
- examples/example2.conf
|
56
|
+
- fluent-plugin-rtf-dash.gemspec
|
57
|
+
- lib/fluent/plugin/out_rtf_dash.rb
|
58
|
+
- test/helper.rb
|
59
|
+
- test/plugin/test_out_rtf_dash.rb
|
60
|
+
homepage: ''
|
61
|
+
licenses:
|
62
|
+
- Apache 2.0
|
63
|
+
metadata: {}
|
64
|
+
post_install_message:
|
65
|
+
rdoc_options: []
|
66
|
+
require_paths:
|
67
|
+
- lib
|
68
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - '>='
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '0'
|
73
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
74
|
+
requirements:
|
75
|
+
- - '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
requirements: []
|
79
|
+
rubyforge_project:
|
80
|
+
rubygems_version: 2.0.3
|
81
|
+
signing_key:
|
82
|
+
specification_version: 4
|
83
|
+
summary: ''
|
84
|
+
test_files:
|
85
|
+
- test/helper.rb
|
86
|
+
- test/plugin/test_out_rtf_dash.rb
|