fluent-plugin-rewrite-tag-filter 0.0.8 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.rdoc CHANGED
@@ -6,29 +6,30 @@
6
6
 
7
7
  It has designed to rewrite tags like mod_rewrite.
8
8
  Also you can change a tag from apache log by domain, status-code(ex. 500 error),
9
- user-agent, and request-uri and so on with regular expression.
9
+ user-agent, request-uri, regex-backreference and so on with regular expression.
10
10
 
11
11
  == Installation
12
12
 
13
- # native fluentd user
13
+ # native gem
14
14
  gem install fluent-plugin-rewrite-tag-filter
15
15
 
16
- # td-agent user
16
+ # td-agent gem
17
17
  /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter
18
18
 
19
19
  == Configuration
20
20
 
21
- Syntax
21
+ === Syntax
22
22
 
23
23
  rewruterule<num:1-200> <attribute> <regex_pattern> <new_tag>
24
24
 
25
- # Option: Capitalize every matched regex backreference (ex: $1, $2).
25
+ # Option: Capitalize every matched regex backreference. (ex: $1, $2)
26
26
  # It has described on usage section below.
27
- capitalize_regex_backreference <default:no>
27
+ capitalize_regex_backreference <yes/no(default)>
28
28
 
29
- Usage
29
+ === Usage
30
+
31
+ It's a sample to exclude some static file log before split tag by domain.
30
32
 
31
- # It's a sample to exclude some static file log before split tag by domain.
32
33
  <source>
33
34
  type tail
34
35
  path /var/log/httpd/access_log
@@ -38,15 +39,15 @@ Usage
38
39
  pos_file /var/log/td-agent/apache_access.pos
39
40
  </source>
40
41
 
42
+ # At rewriterule4, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
41
43
  <match td.apache.access>
42
44
  type rewrite_tag_filter
43
45
  capitalize_regex_backreference yes
44
- rewriterule1 path \.(gif|jpe?g|png|pdf|zip)$ clear
45
- rewriterule2 domain ^maps\.example\.com$ site.ExampleMaps
46
- rewriterule3 domain ^news\.example\.com$ site.ExampleNews
47
- # "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
48
- rewriterule4 domain ^(mail)\.(example)\.com$ site.$2$1
49
- rewriterule5 domain .+ site.unmatched
46
+ rewriterule1 path \.(gif|jpe?g|png|pdf|zip)$ clear
47
+ rewriterule2 domain ^maps\.example\.com$ site.ExampleMaps
48
+ rewriterule3 domain ^news\.example\.com$ site.ExampleNews
49
+ rewriterule4 domain ^(mail)\.(example)\.com$ site.$2$1
50
+ rewriterule5 domain .+ site.unmatched
50
51
  </match>
51
52
 
52
53
  <match site.*>
@@ -63,7 +64,7 @@ Usage
63
64
  type null
64
65
  </match>
65
66
 
66
- Result
67
+ === Result
67
68
 
68
69
  $ mongo
69
70
  MongoDB shell version: 2.2.0
@@ -75,6 +76,18 @@ Result
75
76
  ExampleMail
76
77
  unmatched
77
78
 
79
+ === Debug
80
+
81
+ Logging supported like below.
82
+
83
+ $ tailf /var/log/td-agent/td-agent.log
84
+ 2012-09-16 18:10:51 +0900: adding match pattern="td.apache.access" type="rewrite_tag_filter"
85
+ 2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [1, "path", /\.(gif|jpe?g|png|pdf|zip)$/, "clear"]
86
+ 2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [2, "domain", /^maps\.example\.com$/, "site.ExampleMaps"]
87
+ 2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [3, "domain", /^news\.example\.com$/, "site.ExampleNews"]
88
+ 2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [4, "domain", /^(mail)\.(example)\.com$/, "site.$2$1"]
89
+ 2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [5, "domain", /.+/, "site.unmatched"]
90
+
78
91
  == Example
79
92
 
80
93
  - Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.
@@ -83,11 +96,14 @@ Result
83
96
  - Example2: how to exclude specified patterns before analyze response_time for each virtual domain websites.
84
97
  - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example2.conf
85
98
 
86
- == Review
99
+ == Topic
87
100
 
88
101
  - 自在にタグを書き換える fluent-plugin-rewrite-tag-filter でログ解析が捗るお話 #fluentd
89
102
  - http://d.hatena.ne.jp/yoshi-ken/20120701/1341137269
90
103
 
104
+ - Fluentd & TreasureDataで こっそり始めるログ集計 Fluentd Meetup #2 @mikeda
105
+ - http://www.slideshare.net/baguzy/fluentd-meetup-2-14073930
106
+
91
107
  == TODO
92
108
 
93
109
  - patches welcome!
data/example.conf CHANGED
@@ -21,36 +21,37 @@
21
21
  type copy
22
22
  <store>
23
23
  type rewrite_tag_filter
24
- rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
25
- rewriterule2 domain ^news\.google\.com$ site.GoogleNews
26
- rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
24
+ capitalize_regex_backreference yes
25
+ rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
27
26
  </store>
28
27
  <store>
29
28
  type rewrite_tag_filter
30
- rewriterule1 domain ^maps\.google\.com$ sitepath.GoogleMap
29
+ capitalize_regex_backreference yes
30
+ rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
31
31
  </store>
32
32
  </match>
33
33
 
34
34
 
35
35
  # Second level analyzing
36
- <match sitepath.GoogleMap>
36
+ <match sitepath.GoogleMaps>
37
37
  <store>
38
38
  type rewrite_tag_filter
39
- rewriterule1 path ^/labs site.GoogleMap.Labs
40
- rewriterule2 path ^/static/\d+ site.GoogleMap.Static
41
- rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMap.Help
39
+ rewriterule1 path ^/labs site.GoogleMaps.Labs
40
+ rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
41
+ rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
42
42
  </store>
43
43
  <store>
44
44
  type rewrite_tag_filter
45
- rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMap.referer_YahooHeadlines
46
- rewriterule2 referer news\.livedoor\.com site.GoogleMap.referer_LivedoorNews
45
+ rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
46
+ rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
47
47
  </store>
48
48
  <store>
49
49
  type rewrite_tag_filter
50
- rewriterule1 agent Googlebot/ site.GoogleMap.agent_Googlebot
51
- rewriterule2 agent Googlebot-Mobile/ site.GoogleMap.agent_GooglebotMobile
52
- rewriterule3 agent bingbot site.GoogleMap.agent_Bingbot
53
- rewriterule4 agent Baiduspider site.GoogleMap.agent_Baiduspider
50
+ rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
51
+ rewriterule2 agent " iPhone .+Googlebot-Mobile/" site.GoogleMaps.agent_GooglebotSmartphoneMobile
52
+ rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
53
+ rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
54
+ rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
54
55
  </store>
55
56
  </match>
56
57
 
data/example2.conf CHANGED
@@ -29,10 +29,9 @@
29
29
  rewriterule1 path ^/(img|css|js|static|assets)/ clear
30
30
  rewriterule2 status ^(?!200)$ clear
31
31
  rewriterule3 method ^(?!GET)$ clear
32
- rewriterule4 agent Mac\sOS\sX clear
33
- rewriterule5 agent (spider|bot|crawler|\+http\:) clear
34
- rewriterule6 path ^/(admin|api|backend) site.GoogleMap.backend
35
- rewriterule7 path .+ site.GoogleMap.front
32
+ rewriterule4 agent (spider|bot|crawler|\+http\:) clear
33
+ rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
34
+ rewriterule6 path .+ site.GoogleMap.front
36
35
  </match>
37
36
 
38
37
 
@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-rewrite-tag-filter"
6
- s.version = "0.0.8"
6
+ s.version = "0.0.9"
7
7
  s.authors = ["Kentaro Yoshida"]
8
8
  s.email = ["y.ken.studio@gmail.com"]
9
9
  s.homepage = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
10
- s.summary = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, and request-uri and so on with regular expression.}
10
+ s.summary = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, request-uri, regex-backreference and so on with regular expression.}
11
11
  # s.description = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, and request-uri and so on with regular expression.}
12
12
 
13
13
  s.files = `git ls-files`.split("\n")
@@ -21,13 +21,13 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
21
21
  end
22
22
  (1..PATTERN_MAX_NUM).each do |i|
23
23
  next unless conf["rewriterule#{i}"]
24
- rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].match(/([^ ]+)\s(.+)\s([^ ]+)/).captures
24
+ rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].match(/^([^\s]+)\s+["\/]?(.+?)["\/]?\s+([^\s]+)$/).captures
25
25
  unless regexp != nil && rewritetag != nil
26
26
  raise Fluent::ConfigError, "missing values at rewriterule#{i} " + conf["rewriterule#{i}"].inspect
27
27
  end
28
28
  @rewriterules.push([i, rewritekey, Regexp.new(regexp), rewritetag])
29
29
  rewriterule_names.push(rewritekey + regexp)
30
- $log.info "register rewrite_tag_filter rule: #{@rewriterules.last}"
30
+ $log.info "adding rewrite_tag_filter rule: #{@rewriterules.last}"
31
31
  end
32
32
  rewriterule_index_list = conf.keys.select{|s| s =~ /^rewriterule\d$/}.map{|v| (/^rewriterule(\d)$/.match(v))[1].to_i}
33
33
  unless rewriterule_index_list.reduce(true){|v,i| v and @rewriterules[i - 1]}
@@ -44,14 +44,14 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
44
44
  @rewriterules.each do |index, rewritekey, regexp, rewritetag|
45
45
  rewritevalue = record[rewritekey]
46
46
  next if rewritevalue.nil?
47
- if (regexp && regexp.match(rewritevalue))
48
- rewrite = true
49
- tag = rewritetag
50
- if rewritetag.include?('$')
51
- tag = rewritetag.gsub(/\$\d+/, map_regex_table($~.captures))
52
- end
53
- break
54
- end
47
+ next unless (regexp && regexp.match(rewritevalue))
48
+ rewrite = true
49
+ tag = if rewritetag.include?('$')
50
+ rewritetag.gsub(/\$\d+/, map_regex_table($~.captures))
51
+ else
52
+ rewritetag
53
+ end
54
+ break
55
55
  end
56
56
  Fluent::Engine.emit(tag, time, record) if (rewrite)
57
57
  end
@@ -12,12 +12,14 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
12
12
  rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
13
13
  ]
14
14
 
15
+ # aggresive test
16
+ # indentation, comment, capitalize_regex_backreference, regex with space aside, slash/quote delimited
15
17
  CONFIG2 = %[
16
18
  capitalize_regex_backreference yes
17
- rewriterule1 domain ^www\.google\.com$ site.Google
18
- rewriterule2 domain ^(news)\.(google)\.com$ site.$2$1
19
- rewriterule3 agent Mac\sOS\sX agent.MacOSX
20
- rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
19
+ rewriterule1 domain "^www\.google\.com$" site.Google # some comment
20
+ rewriterule2 domain /^(news)\.(google)\.com$/ site.$2$1
21
+ rewriterule3 agent / Mac OS X / agent.MacOSX
22
+ rewriterule4 agent /(Googlebot|CustomBot)-([a-zA-Z]+)/ agent.$1-$2
21
23
  ]
22
24
 
23
25
  def create_driver(conf=CONFIG,tag='test')
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-rewrite-tag-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.8
4
+ version: 0.0.9
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-09-15 00:00:00.000000000 Z
12
+ date: 2012-09-16 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
16
- requirement: &20127680 !ruby/object:Gem::Requirement
16
+ requirement: &10180420 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :development
23
23
  prerelease: false
24
- version_requirements: *20127680
24
+ version_requirements: *10180420
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: fluentd
27
- requirement: &20126860 !ruby/object:Gem::Requirement
27
+ requirement: &10179520 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *20126860
35
+ version_requirements: *10179520
36
36
  description:
37
37
  email:
38
38
  - y.ken.studio@gmail.com
@@ -76,7 +76,7 @@ signing_key:
76
76
  specification_version: 3
77
77
  summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.
78
78
  Also you can change a tag from apache log by domain, status-code(ex. 500 error),
79
- user-agent, and request-uri and so on with regular expression.
79
+ user-agent, request-uri, regex-backreference and so on with regular expression.
80
80
  test_files:
81
81
  - test/helper.rb
82
82
  - test/plugin/test_out_rewrite_tag_filter.rb