fluent-plugin-rewrite-tag-filter 0.0.2 → 0.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. data/README.rdoc +8 -2
  2. data/example.conf +58 -16
  3. data/example2.conf +79 -0
  4. data/fluent-plugin-rewrite-tag-filter.gemspec +3 -3
  5. metadata +9 -8
data/README.rdoc CHANGED
@@ -18,7 +18,7 @@ Modify Tags with data matches any of specified regexp patterns in specified attr
18
18
 
19
19
  Format
20
20
 
21
- rewruterule<num:1-200> <attribute> <regex_pattern> <rewrite_tag>
21
+ rewruterule<num:1-200> <attribute> <regex_pattern> <new_tag>
22
22
 
23
23
  Usage
24
24
 
@@ -34,15 +34,20 @@ Usage
34
34
  host 10.10.0.36
35
35
  database apache_access
36
36
  remove_tag_prefix site
37
+ tag_mapped
38
+ collection misc
37
39
  capped
38
40
  capped_size 100m
39
41
  </match>
40
42
 
41
- Example
43
+ == Example
42
44
 
43
45
  - Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.
44
46
  - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example.conf
45
47
 
48
+ - Example2: how to exclude specified patterns before analyze response_time for each virtual domain websites.
49
+ - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example2.conf
50
+
46
51
  == TODO
47
52
 
48
53
  - patches welcome!
@@ -51,3 +56,4 @@ Example
51
56
 
52
57
  Copyright:: Copyright (c) 2012- Kentaro Yoshida
53
58
  License:: Apache License, Version 2.0
59
+
data/example.conf CHANGED
@@ -1,4 +1,10 @@
1
- # it is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.
1
+ # It is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.
2
+
3
+ # appearing plugins:
4
+ # rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
5
+ # forest: http://rubygems.org/gems/fluent-plugin-forest
6
+ # datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
7
+ # growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
2
8
 
3
9
  <source>
4
10
  type tail
@@ -9,13 +15,47 @@
9
15
  pos_file /var/log/td-agent/apache_access.pos
10
16
  </source>
11
17
 
18
+
19
+ # Extract specified virtual domain
12
20
  <match td.apache.access>
13
- type rewrite_tag_filter
14
- rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
15
- rewriterule2 domain ^news\.google\.com$ site.GoogleNews
16
- rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
21
+ type copy
22
+ <store>
23
+ type rewrite_tag_filter
24
+ rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
25
+ rewriterule2 domain ^news\.google\.com$ site.GoogleNews
26
+ rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
27
+ </store>
28
+ <store>
29
+ type rewrite_tag_filter
30
+ rewriterule1 domain ^maps\.google\.com$ sitepath.GoogleMap
31
+ </store>
17
32
  </match>
18
33
 
34
+
35
+ # Second level analyzing
36
+ <match sitepath.GoogleMap>
37
+ <store>
38
+ type rewrite_tag_filter
39
+ rewriterule1 path ^/labs site.GoogleMap.Labs
40
+ rewriterule2 path ^/static/\d+ site.GoogleMap.Static
41
+ rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMap.Help
42
+ </store>
43
+ <store>
44
+ type rewrite_tag_filter
45
+ rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMap.referer_YahooHeadlines
46
+ rewriterule2 referer news\.livedoor\.com site.GoogleMap.referer_LivedoorNews
47
+ </store>
48
+ <store>
49
+ type rewrite_tag_filter
50
+ rewriterule1 agent Googlebot/ site.GoogleMap.agent_Googlebot
51
+ rewriterule2 agent Googlebot-Mobile/ site.GoogleMap.agent_GooglebotMobile
52
+ rewriterule3 agent bingbot site.GoogleMap.agent_Bingbot
53
+ rewriterule4 agent Baiduspider site.GoogleMap.agent_Baiduspider
54
+ </store>
55
+ </match>
56
+
57
+
58
+ # Summarize
19
59
  <match site.**>
20
60
  type copy
21
61
  <store>
@@ -75,27 +115,29 @@
75
115
  </store>
76
116
  </match>
77
117
 
78
- <match gf.responsecode.**>
118
+
119
+ # Graph
120
+ <match gf.responsetime.**>
79
121
  type forest
80
122
  subtype growthforecast
81
- remove_prefix gf.responsecode.site
123
+ remove_prefix gf.responsetime.site
82
124
  <template>
83
- gfapi_url http://10.10.0.55:5125/api/
125
+ gfapi_url http://localhost:5125/api/
84
126
  service __TAG__
85
- section response_code
86
- name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
127
+ section response_time
128
+ name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
87
129
  </template>
88
130
  </match>
89
131
 
90
- <match gf.responsetime.**>
132
+ <match gf.responsecode.**>
91
133
  type forest
92
134
  subtype growthforecast
93
- remove_prefix gf.responsetime.site
135
+ remove_prefix gf.responsecode.site
94
136
  <template>
95
- gfapi_url http://10.10.0.55:5125/api/
137
+ gfapi_url http://localhost:5125/api/
96
138
  service __TAG__
97
- section response_time
98
- name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
139
+ section response_code
140
+ name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
99
141
  </template>
100
142
  </match>
101
143
 
@@ -104,7 +146,7 @@
104
146
  subtype growthforecast
105
147
  remove_prefix gf.useragent.site
106
148
  <template>
107
- gfapi_url http://10.10.0.55:5125/api/
149
+ gfapi_url http://localhost:5125/api/
108
150
  service __TAG__
109
151
  section useragent
110
152
  name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
data/example2.conf ADDED
@@ -0,0 +1,79 @@
1
+ # It is a sample how to exclude specified patterns before analyze response_time for each virtual domain websites.
2
+
3
+ # appearing plugins:
4
+ # rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
5
+ # forest: http://rubygems.org/gems/fluent-plugin-forest
6
+ # datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
7
+ # growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
8
+
9
+ <source>
10
+ type tail
11
+ path /var/log/httpd/access_log
12
+ format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
13
+ time_format %d/%b/%Y:%H:%M:%S %z
14
+ tag td.apache.access
15
+ pos_file /var/log/td-agent/apache_access.pos
16
+ </source>
17
+
18
+
19
+ # Extract specified virtual domain
20
+ <match td.apache.access>
21
+ type rewrite_tag_filter
22
+ rewriterule1 domain ^maps\.google\.com$ filter.GoogleMap
23
+ </match>
24
+
25
+
26
+ # Filtering
27
+ <match filter.GoogleMap>
28
+ <store>
29
+ type rewrite_tag_filter
30
+ rewriterule1 path ^/(img|css|js|static|assets)/ clear
31
+ rewriterule2 status ^(?!200)$ clear
32
+ rewriterule3 method ^(?!GET)$ clear
33
+ rewriterule4 agent (spider|bot|crawler|\+http\:) clear
34
+ rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
35
+ rewriterule6 path .+ site.GoogleMap.front
36
+ </store>
37
+ </match>
38
+
39
+
40
+ # Summarize
41
+ <match site.**>
42
+ type forest
43
+ subtype datacounter
44
+ <template>
45
+ unit minute
46
+ count_key response_time
47
+ outcast_unmatched false
48
+ aggregate all
49
+ tag gf.responsetime.__TAG__
50
+ pattern1 0-100msec ^\d{1,5}$
51
+ pattern2 100-300msec ^[1-2]\d{5}$
52
+ pattern3 300-600msec ^[3-5]\d{5}$
53
+ pattern4 600msec-1sec ^[6-9]\d{5}$
54
+ pattern5 1-2sec ^1\d{6}$
55
+ pattern6 2-10sec ^[2-9]\d{6}$
56
+ pattern7 10sec_over ^\d{8,}$
57
+ </template>
58
+ </match>
59
+
60
+
61
+ # Graph
62
+ <match gf.responsetime.**>
63
+ type forest
64
+ subtype growthforecast
65
+ remove_prefix gf.responsetime.site
66
+ <template>
67
+ gfapi_url http://localhost:5125/api/
68
+ service __TAG__
69
+ section response_time
70
+ name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
71
+ </template>
72
+ </match>
73
+
74
+
75
+ # Clear tag
76
+ <match clear>
77
+ type null
78
+ </match>
79
+
data/fluent-plugin-rewrite-tag-filter.gemspec CHANGED
@@ -3,12 +3,12 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-rewrite-tag-filter"
6
- s.version = "0.0.2"
6
+ s.version = "0.0.3"
7
7
  s.authors = ["Kentaro Yoshida"]
8
8
  s.email = ["y.ken.studio@gmail.com"]
9
9
  s.homepage = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
10
- s.summary = %q{Output filter plugin to rewrite tags that matches specified attribute}
11
- s.description = %q{Modify Tags with data matches any of specified regexp patterns in specified attribute.}
10
+ s.summary = %q{Fluentd Output filter plugin to rewrite tags that matches specified attribute.}
11
+ s.description = %q{Rewrite Tags with data matches any of specified regexp patterns in specified attribute.}
12
12
 
13
13
  s.rubyforge_project = "fluent-plugin-rewrite-tag-filter"
14
14
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-rewrite-tag-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-06-30 00:00:00.000000000 Z
12
+ date: 2012-07-01 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
16
- requirement: &11789920 !ruby/object:Gem::Requirement
16
+ requirement: &17995380 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :development
23
23
  prerelease: false
24
- version_requirements: *11789920
24
+ version_requirements: *17995380
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: fluentd
27
- requirement: &11788560 !ruby/object:Gem::Requirement
27
+ requirement: &17994080 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,8 +32,8 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *11788560
36
- description: Modify Tags with data matches any of specified regexp patterns in specified
35
+ version_requirements: *17994080
36
+ description: Rewrite Tags with data matches any of specified regexp patterns in specified
37
37
  attribute.
38
38
  email:
39
39
  - y.ken.studio@gmail.com
@@ -47,6 +47,7 @@ files:
47
47
  - README.rdoc
48
48
  - Rakefile
49
49
  - example.conf
50
+ - example2.conf
50
51
  - fluent-plugin-rewrite-tag-filter.gemspec
51
52
  - lib/fluent/plugin/out_rewrite_tag_filter.rb
52
53
  - test/helper.rb
@@ -74,7 +75,7 @@ rubyforge_project: fluent-plugin-rewrite-tag-filter
74
75
  rubygems_version: 1.8.11
75
76
  signing_key:
76
77
  specification_version: 3
77
- summary: Output filter plugin to rewrite tags that matches specified attribute
78
+ summary: Fluentd Output filter plugin to rewrite tags that matches specified attribute.
78
79
  test_files:
79
80
  - test/helper.rb
80
81
  - test/plugin/test_out_rewrite_tag_filter.rb