fluent-plugin-rewrite-tag-filter 0.0.8 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +32 -16
- data/example.conf +15 -14
- data/example2.conf +3 -4
- data/fluent-plugin-rewrite-tag-filter.gemspec +2 -2
- data/lib/fluent/plugin/out_rewrite_tag_filter.rb +10 -10
- data/test/plugin/test_out_rewrite_tag_filter.rb +6 -4
- metadata +7 -7
data/README.rdoc
CHANGED
@@ -6,29 +6,30 @@
|
|
6
6
|
|
7
7
|
It has designed to rewrite tags like mod_rewrite.
|
8
8
|
Also you can change a tag from apache log by domain, status-code(ex. 500 error),
|
9
|
-
user-agent,
|
9
|
+
user-agent, request-uri, regex-backreference and so on with regular expression.
|
10
10
|
|
11
11
|
== Installation
|
12
12
|
|
13
|
-
# native
|
13
|
+
# native gem
|
14
14
|
gem install fluent-plugin-rewrite-tag-filter
|
15
15
|
|
16
|
-
# td-agent
|
16
|
+
# td-agent gem
|
17
17
|
/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter
|
18
18
|
|
19
19
|
== Configuration
|
20
20
|
|
21
|
-
Syntax
|
21
|
+
=== Syntax
|
22
22
|
|
23
23
|
rewruterule<num:1-200> <attribute> <regex_pattern> <new_tag>
|
24
24
|
|
25
|
-
# Option: Capitalize every matched regex backreference (ex: $1, $2)
|
25
|
+
# Option: Capitalize every matched regex backreference. (ex: $1, $2)
|
26
26
|
# It has described on usage section below.
|
27
|
-
capitalize_regex_backreference <default
|
27
|
+
capitalize_regex_backreference <yes/no(default)>
|
28
28
|
|
29
|
-
Usage
|
29
|
+
=== Usage
|
30
|
+
|
31
|
+
It's a sample to exclude some static file log before split tag by domain.
|
30
32
|
|
31
|
-
# It's a sample to exclude some static file log before split tag by domain.
|
32
33
|
<source>
|
33
34
|
type tail
|
34
35
|
path /var/log/httpd/access_log
|
@@ -38,15 +39,15 @@ Usage
|
|
38
39
|
pos_file /var/log/td-agent/apache_access.pos
|
39
40
|
</source>
|
40
41
|
|
42
|
+
# At rewriterule4, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
|
41
43
|
<match td.apache.access>
|
42
44
|
type rewrite_tag_filter
|
43
45
|
capitalize_regex_backreference yes
|
44
|
-
rewriterule1 path
|
45
|
-
rewriterule2 domain ^maps\.example\.com$
|
46
|
-
rewriterule3 domain ^news\.example\.com$
|
47
|
-
|
48
|
-
|
49
|
-
rewriterule5 domain .+ site.unmatched
|
46
|
+
rewriterule1 path \.(gif|jpe?g|png|pdf|zip)$ clear
|
47
|
+
rewriterule2 domain ^maps\.example\.com$ site.ExampleMaps
|
48
|
+
rewriterule3 domain ^news\.example\.com$ site.ExampleNews
|
49
|
+
rewriterule4 domain ^(mail)\.(example)\.com$ site.$2$1
|
50
|
+
rewriterule5 domain .+ site.unmatched
|
50
51
|
</match>
|
51
52
|
|
52
53
|
<match site.*>
|
@@ -63,7 +64,7 @@ Usage
|
|
63
64
|
type null
|
64
65
|
</match>
|
65
66
|
|
66
|
-
Result
|
67
|
+
=== Result
|
67
68
|
|
68
69
|
$ mongo
|
69
70
|
MongoDB shell version: 2.2.0
|
@@ -75,6 +76,18 @@ Result
|
|
75
76
|
ExampleMail
|
76
77
|
unmatched
|
77
78
|
|
79
|
+
=== Debug
|
80
|
+
|
81
|
+
Logging supported like below.
|
82
|
+
|
83
|
+
$ tailf /var/log/td-agent/td-agent.log
|
84
|
+
2012-09-16 18:10:51 +0900: adding match pattern="td.apache.access" type="rewrite_tag_filter"
|
85
|
+
2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [1, "path", /\.(gif|jpe?g|png|pdf|zip)$/, "clear"]
|
86
|
+
2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [2, "domain", /^maps\.example\.com$/, "site.ExampleMaps"]
|
87
|
+
2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [3, "domain", /^news\.example\.com$/, "site.ExampleNews"]
|
88
|
+
2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [4, "domain", /^(mail)\.(example)\.com$/, "site.$2$1"]
|
89
|
+
2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [5, "domain", /.+/, "site.unmatched"]
|
90
|
+
|
78
91
|
== Example
|
79
92
|
|
80
93
|
- Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.
|
@@ -83,11 +96,14 @@ Result
|
|
83
96
|
- Example2: how to exclude specified patterns before analyze response_time for each virtual domain websites.
|
84
97
|
- https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example2.conf
|
85
98
|
|
86
|
-
==
|
99
|
+
== Topic
|
87
100
|
|
88
101
|
- 自在にタグを書き換える fluent-plugin-rewrite-tag-filter でログ解析が捗るお話 #fluentd
|
89
102
|
- http://d.hatena.ne.jp/yoshi-ken/20120701/1341137269
|
90
103
|
|
104
|
+
- Fluentd & TreasureDataで こっそり始めるログ集計 Fluentd Meetup #2 @mikeda
|
105
|
+
- http://www.slideshare.net/baguzy/fluentd-meetup-2-14073930
|
106
|
+
|
91
107
|
== TODO
|
92
108
|
|
93
109
|
- patches welcome!
|
data/example.conf
CHANGED
@@ -21,36 +21,37 @@
|
|
21
21
|
type copy
|
22
22
|
<store>
|
23
23
|
type rewrite_tag_filter
|
24
|
-
|
25
|
-
|
26
|
-
rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
|
24
|
+
capitalize_regex_backreference yes
|
25
|
+
rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
|
27
26
|
</store>
|
28
27
|
<store>
|
29
28
|
type rewrite_tag_filter
|
30
|
-
|
29
|
+
capitalize_regex_backreference yes
|
30
|
+
rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
|
31
31
|
</store>
|
32
32
|
</match>
|
33
33
|
|
34
34
|
|
35
35
|
# Second level analyzing
|
36
|
-
<match sitepath.
|
36
|
+
<match sitepath.GoogleMaps>
|
37
37
|
<store>
|
38
38
|
type rewrite_tag_filter
|
39
|
-
rewriterule1 path ^/labs site.
|
40
|
-
rewriterule2 path ^/static/\d+ site.
|
41
|
-
rewriterule3 path ^/help/maps/(streetview|getmaps) site.
|
39
|
+
rewriterule1 path ^/labs site.GoogleMaps.Labs
|
40
|
+
rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
|
41
|
+
rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
|
42
42
|
</store>
|
43
43
|
<store>
|
44
44
|
type rewrite_tag_filter
|
45
|
-
rewriterule1 referer headlines\.yahoo\.co\.jp site.
|
46
|
-
rewriterule2 referer news\.livedoor\.com site.
|
45
|
+
rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
|
46
|
+
rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
|
47
47
|
</store>
|
48
48
|
<store>
|
49
49
|
type rewrite_tag_filter
|
50
|
-
rewriterule1 agent Googlebot/ site.
|
51
|
-
rewriterule2 agent Googlebot-Mobile/ site.
|
52
|
-
rewriterule3 agent
|
53
|
-
rewriterule4 agent
|
50
|
+
rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
|
51
|
+
rewriterule2 agent " iPhone .+Googlebot-Mobile/" site.GoogleMaps.agent_GooglebotSmartphoneMobile
|
52
|
+
rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
|
53
|
+
rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
|
54
|
+
rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
|
54
55
|
</store>
|
55
56
|
</match>
|
56
57
|
|
data/example2.conf
CHANGED
@@ -29,10 +29,9 @@
|
|
29
29
|
rewriterule1 path ^/(img|css|js|static|assets)/ clear
|
30
30
|
rewriterule2 status ^(?!200)$ clear
|
31
31
|
rewriterule3 method ^(?!GET)$ clear
|
32
|
-
rewriterule4 agent
|
33
|
-
rewriterule5
|
34
|
-
rewriterule6 path
|
35
|
-
rewriterule7 path .+ site.GoogleMap.front
|
32
|
+
rewriterule4 agent (spider|bot|crawler|\+http\:) clear
|
33
|
+
rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
|
34
|
+
rewriterule6 path .+ site.GoogleMap.front
|
36
35
|
</match>
|
37
36
|
|
38
37
|
|
@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-rewrite-tag-filter"
|
6
|
-
s.version = "0.0.
|
6
|
+
s.version = "0.0.9"
|
7
7
|
s.authors = ["Kentaro Yoshida"]
|
8
8
|
s.email = ["y.ken.studio@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
|
10
|
-
s.summary = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent,
|
10
|
+
s.summary = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, request-uri, regex-backreference and so on with regular expression.}
|
11
11
|
# s.description = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, and request-uri and so on with regular expression.}
|
12
12
|
|
13
13
|
s.files = `git ls-files`.split("\n")
|
@@ -21,13 +21,13 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
21
21
|
end
|
22
22
|
(1..PATTERN_MAX_NUM).each do |i|
|
23
23
|
next unless conf["rewriterule#{i}"]
|
24
|
-
rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].match(
|
24
|
+
rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].match(/^([^\s]+)\s+["\/]?(.+?)["\/]?\s+([^\s]+)$/).captures
|
25
25
|
unless regexp != nil && rewritetag != nil
|
26
26
|
raise Fluent::ConfigError, "missing values at rewriterule#{i} " + conf["rewriterule#{i}"].inspect
|
27
27
|
end
|
28
28
|
@rewriterules.push([i, rewritekey, Regexp.new(regexp), rewritetag])
|
29
29
|
rewriterule_names.push(rewritekey + regexp)
|
30
|
-
$log.info "
|
30
|
+
$log.info "adding rewrite_tag_filter rule: #{@rewriterules.last}"
|
31
31
|
end
|
32
32
|
rewriterule_index_list = conf.keys.select{|s| s =~ /^rewriterule\d$/}.map{|v| (/^rewriterule(\d)$/.match(v))[1].to_i}
|
33
33
|
unless rewriterule_index_list.reduce(true){|v,i| v and @rewriterules[i - 1]}
|
@@ -44,14 +44,14 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
44
44
|
@rewriterules.each do |index, rewritekey, regexp, rewritetag|
|
45
45
|
rewritevalue = record[rewritekey]
|
46
46
|
next if rewritevalue.nil?
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
47
|
+
next unless (regexp && regexp.match(rewritevalue))
|
48
|
+
rewrite = true
|
49
|
+
tag = if rewritetag.include?('$')
|
50
|
+
rewritetag.gsub(/\$\d+/, map_regex_table($~.captures))
|
51
|
+
else
|
52
|
+
rewritetag
|
53
|
+
end
|
54
|
+
break
|
55
55
|
end
|
56
56
|
Fluent::Engine.emit(tag, time, record) if (rewrite)
|
57
57
|
end
|
@@ -12,12 +12,14 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
12
12
|
rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
|
13
13
|
]
|
14
14
|
|
15
|
+
# aggresive test
|
16
|
+
# indentation, comment, capitalize_regex_backreference, regex with space aside, slash/quote delimited
|
15
17
|
CONFIG2 = %[
|
16
18
|
capitalize_regex_backreference yes
|
17
|
-
rewriterule1 domain ^www\.google\.com$
|
18
|
-
rewriterule2 domain
|
19
|
-
rewriterule3 agent Mac
|
20
|
-
rewriterule4 agent
|
19
|
+
rewriterule1 domain "^www\.google\.com$" site.Google # some comment
|
20
|
+
rewriterule2 domain /^(news)\.(google)\.com$/ site.$2$1
|
21
|
+
rewriterule3 agent / Mac OS X / agent.MacOSX
|
22
|
+
rewriterule4 agent /(Googlebot|CustomBot)-([a-zA-Z]+)/ agent.$1-$2
|
21
23
|
]
|
22
24
|
|
23
25
|
def create_driver(conf=CONFIG,tag='test')
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rewrite-tag-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-09-
|
12
|
+
date: 2012-09-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
16
|
-
requirement: &
|
16
|
+
requirement: &10180420 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *10180420
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: fluentd
|
27
|
-
requirement: &
|
27
|
+
requirement: &10179520 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,7 +32,7 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *10179520
|
36
36
|
description:
|
37
37
|
email:
|
38
38
|
- y.ken.studio@gmail.com
|
@@ -76,7 +76,7 @@ signing_key:
|
|
76
76
|
specification_version: 3
|
77
77
|
summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.
|
78
78
|
Also you can change a tag from apache log by domain, status-code(ex. 500 error),
|
79
|
-
user-agent,
|
79
|
+
user-agent, request-uri, regex-backreference and so on with regular expression.
|
80
80
|
test_files:
|
81
81
|
- test/helper.rb
|
82
82
|
- test/plugin/test_out_rewrite_tag_filter.rb
|