fluent-plugin-rewrite-tag-filter 2.0.0 → 2.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +30 -16
  3. data/example.conf +132 -96
  4. data/example2.conf +63 -37
  5. data/fluent-plugin-rewrite-tag-filter.gemspec +3 -2
  6. data/lib/fluent/plugin/out_rewrite_tag_filter.rb +9 -7
  7. data/test/plugin/test_out_rewrite_tag_filter.rb +46 -0
  8. metadata +25 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 2644562a37644d56fa8712f13d46aa0ff42cb9ec
4
- data.tar.gz: 22ac2b253497f823babee508e41d695b9bdc8218
2
+ SHA256:
3
+ metadata.gz: 9a976b12b3e1ab4b56116a3690d0c48949ff34788f1b165b39909c52d4e1983d
4
+ data.tar.gz: 435ca4ea943d418f2803ee787422271850ededecbe20bd82efa518bb5ff12a7a
5
5
  SHA512:
6
- metadata.gz: eaa2ee232b41ba0b6f387adb69621e78a411919d8d074c3ce014d344b4211a1dad7d47b61a06698cbc4fa0791de55a87111bf91383c47cee3f56f71299ad53e1
7
- data.tar.gz: ff33f9efd2106fc70e85023a139563764aa8414329066459bd9dd177133ee9ff8b9bff20119a50d8621c20573f3d7cdcfc3cb37c1d67a4f6210190b9523bb878
6
+ metadata.gz: 49b4bf86556799b2ac17c37e214754af8ea80489d3817a7026e4aac725ac360605154538e885f4c5cc6843424a1d9575ea383a4fa99c6e3b045169347855ecd2
7
+ data.tar.gz: 7f34dc3e1f1d5178051d3e2af5d433064bb8219638f4181854c6e712341fa0bdbe52f8baa527e9b66967c79686ea2eccf0641fcaadfce22379e47b9bb7747c59
data/README.md CHANGED
@@ -7,6 +7,8 @@ Re-emit the record with rewrited tag when a value matches/unmatches with a regul
7
7
  Also you can change a tag from Apache log by domain, status code (ex. 500 error),
8
8
  user-agent, request-uri, regex-backreference and so on with regular expression.
9
9
 
10
+ This is an output plugin because fluentd's `filter` doesn't allow tag rewrite.
11
+
10
12
  ## Requirements
11
13
 
12
14
  | fluent-plugin-rewrite-tag-filter | Fluentd | Ruby |
@@ -16,23 +18,25 @@ user-agent, request-uri, regex-backreference and so on with regular expression.
16
18
 
17
19
  ## Installation
18
20
 
19
- Install with `gem`, `fluent-gem` or `td-agent-gem` command as:
21
+ Install with `gem` or `td-agent-gem` command as:
20
22
 
21
23
  ```
22
24
  # for system installed fluentd
23
25
  $ gem install fluent-plugin-rewrite-tag-filter
24
26
 
25
- # for td-agent (Legacy)
26
- $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
27
-
28
27
  # for td-agent2 (with fluentd v0.12)
29
- $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
28
+ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.6.0
29
+
30
+ # for td-agent3 (with fluentd v0.14)
31
+ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter
30
32
  ```
31
33
 
34
+ For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/articles/plugin-management)
35
+
32
36
  ## Configuration
33
37
 
34
38
  * **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
35
- * Deprecated: Use <rule> section
39
+ * Obsoleted: Use <rule> section
36
40
  * **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
37
41
  * Default value: no
38
42
  * **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
@@ -42,7 +46,9 @@ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
42
46
  ### \<rule\> section (optional) (multiple)
43
47
 
44
48
  * **key** (string) (required): The field name to which the regular expression is applied
45
- * **pattern** (regexp) (required): The regular expression
49
+ * **pattern** (regexp) (required): The regular expression.
50
+ `/regexp/` is preferred because `/regexp/` style can support character classes such as `/[a-z]/`.
51
+ The pattern without slashes will cause errors if you use patterns start with character classes.
46
52
  * **tag** (string) (required): New tag
47
53
  * **invert** (bool) (optional): If true, rewrite tag when unmatch pattern
48
54
  * Default value: `false`
@@ -70,39 +76,39 @@ It's a sample to exclude some static file log before split tag by domain.
70
76
  capitalize_regex_backreference yes
71
77
  <rule>
72
78
  key path
73
- pattern \.(gif|jpe?g|png|pdf|zip)$
79
+ pattern /\.(gif|jpe?g|png|pdf|zip)$/
74
80
  tag clear
75
81
  </rule>
76
82
  <rule>
77
83
  key status
78
- pattern ^200$
84
+ pattern /^200$/
79
85
  tag clear
80
86
  invert true
81
87
  </rule>
82
88
  <rule>
83
89
  key domain
84
- pattern ^.+\.com$
90
+ pattern /^.+\.com$/
85
91
  tag clear
86
92
  invert true
87
93
  </rule>
88
94
  <rule>
89
95
  key domain
90
- pattern ^maps\.example\.com$
96
+ pattern /^maps\.example\.com$/
91
97
  tag site.ExampleMaps
92
98
  </rule>
93
99
  <rule>
94
100
  key domain
95
- pattern ^news\.example\.com$
101
+ pattern /^news\.example\.com$/
96
102
  tag site.ExampleNews
97
103
  </rule>
98
104
  <rule>
99
105
  key domain
100
- pattern ^(mail)\.(example)\.com$
106
+ pattern /^(mail)\.(example)\.com$/
101
107
  tag site.$2$1
102
108
  </rule>
103
109
  <rule>
104
110
  key domain
105
- pattern .+
111
+ pattern /.+/
106
112
  tag site.unmatched
107
113
  </rule>
108
114
  </match>
@@ -157,7 +163,11 @@ Dot notation:
157
163
  ```
158
164
  <match kubernetes.**>
159
165
  @type rewrite_tag_filter
160
- rewriterule1 $.kubernetes.namespace_name ^(.+)$ $1.${tag}
166
+ <rule>
167
+ key $.kubernetes.namespace_name
168
+ pattern ^(.+)$
169
+ tag $1.${tag}
170
+ </rule>
161
171
  </match>
162
172
  ```
163
173
 
@@ -166,7 +176,11 @@ Bracket notation:
166
176
  ```
167
177
  <match kubernetes.**>
168
178
  @type rewrite_tag_filter
169
- rewriterule1 $['kubernetes']['namespace_name'] ^(.+)$ $1.${tag}
179
+ <rule>
180
+ key $['kubernetes']['namespace_name']
181
+ pattern ^(.+)$
182
+ tag $1.${tag}
183
+ </rule>
170
184
  </match>
171
185
  ```
172
186
 
data/example.conf CHANGED
@@ -2,15 +2,17 @@
2
2
 
3
3
  # appearing plugins:
4
4
  # rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
5
- # forest: http://rubygems.org/gems/fluent-plugin-forest
6
5
  # datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
7
6
  # growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
8
7
 
9
8
  <source>
10
- type tail
9
+ @type tail
11
10
  path /var/log/httpd/access_log
12
- format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
13
- time_format %d/%b/%Y:%H:%M:%S %z
11
+ <parse>
12
+ @type regexp
13
+ expression /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
14
+ time_format %d/%b/%Y:%H:%M:%S %z
15
+ </parse>
14
16
  tag td.apache.access
15
17
  pos_file /var/log/td-agent/apache_access.pos
16
18
  </source>
@@ -18,139 +20,173 @@
18
20
 
19
21
  # Extract specified virtual domain
20
22
  <match td.apache.access>
21
- type copy
23
+ @type copy
22
24
  <store>
23
- type rewrite_tag_filter
25
+ @type rewrite_tag_filter
24
26
  capitalize_regex_backreference yes
25
- rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
27
+ <rule>
28
+ key domain
29
+ pattern /^(maps|news|mail)\.google\.com$/
30
+ tag site.Google$1
31
+ </rule>
26
32
  </store>
27
33
  <store>
28
- type rewrite_tag_filter
34
+ @type rewrite_tag_filter
29
35
  capitalize_regex_backreference yes
30
- rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
36
+ <rule>
37
+ key domain
38
+ pattern /^(maps)\.google\.com$/
39
+ tag sitepath.Google$1
40
+ </rule>
31
41
  </store>
32
42
  </match>
33
43
 
34
44
 
35
45
  # Second level analyzing
36
46
  <match sitepath.GoogleMaps>
47
+ @type copy
37
48
  <store>
38
- type rewrite_tag_filter
39
- rewriterule1 path ^/labs site.GoogleMaps.Labs
40
- rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
41
- rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
49
+ @type rewrite_tag_filter
50
+ <rule>
51
+ key path
52
+ pattern /^\/labs/
53
+ tag site.GoogleMaps.Labs
54
+ </rule>
55
+ <rule>
56
+ key path
57
+ pattern /^\/static/\d+/
58
+ tag site.GoogleMaps.Static
59
+ </rule>
60
+ <rule>
61
+ key path
62
+ pattern /^\/help\/maps\/(streetview|getmaps)/
63
+ tag site.GoogleMaps.Help
64
+ </rule>
42
65
  </store>
43
66
  <store>
44
- type rewrite_tag_filter
45
- rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
46
- rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
67
+ @type rewrite_tag_filter
68
+ <rule>
69
+ key referer
70
+ pattern /headlines\.yahoo\.co\.jp/
71
+ tag site.GoogleMaps.referer_YahooHeadlines
72
+ </rule>
73
+ <rule>
74
+ key referer
75
+ pattern /news\.livedoor\.com/
76
+ tag site.GoogleMaps.referer_LivedoorNews
77
+ </rule>
47
78
  </store>
48
79
  <store>
49
- type rewrite_tag_filter
50
- rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
51
- rewriterule2 agent ^.* iPhone .+Googlebot-Mobile/.*$ site.GoogleMaps.agent_GooglebotSmartphone
52
- rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
53
- rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
54
- rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
80
+ @type rewrite_tag_filter
81
+ <rule>
82
+ key agent
83
+ pattern /Googlebot\//
84
+ tag site.GoogleMaps.agent_Googlebot
85
+ </rule>
86
+ <rule>
87
+ key agent
88
+ pattern /^.* iPhone .+Googlebot-Mobile\/.*$/
89
+ tag site.GoogleMaps.agent_GooglebotSmartphone
90
+ </rule>
91
+ <rule>
92
+ key agent
93
+ pattern /Googlebot-Mobile\//
94
+ tag site.GoogleMaps.agent_GooglebotMobile
95
+ </rule>
96
+ <rule>
97
+ key agent
98
+ pattern /bingbot/
99
+ tag site.GoogleMaps.agent_Bingbot
100
+ </rule>
101
+ <rule>
102
+ key agent
103
+ pattern /Baiduspider/
104
+ tag site.GoogleMaps.agent_Baiduspider
105
+ </rule>
55
106
  </store>
56
107
  </match>
57
108
 
58
109
 
59
110
  # Summarize
60
111
  <match site.**>
61
- type copy
112
+ @type copy
62
113
  <store>
63
- type forest
64
- subtype datacounter
65
- <template>
66
- unit minute
67
- count_key response_time
68
- outcast_unmatched false
69
- aggregate all
70
- tag gf.responsetime.__TAG__
71
- pattern1 0-100msec ^\d{1,5}$
72
- pattern2 100-300msec ^[1-2]\d{5}$
73
- pattern3 300-600msec ^[3-5]\d{5}$
74
- pattern4 600msec-1sec ^[6-9]\d{5}$
75
- pattern5 1-2sec ^1\d{6}$
76
- pattern6 2-10sec ^[2-9]\d{6}$
77
- pattern7 10sec_over ^\d{8,}$
78
- </template>
114
+ @type datacounter
115
+ unit minute
116
+ count_key response_time
117
+ outcast_unmatched false
118
+ aggregate tag
119
+ tag_prefix gf.responsetime
120
+ output_per_tag yes
121
+ pattern1 0-100msec ^\d{1,5}$
122
+ pattern2 100-300msec ^[1-2]\d{5}$
123
+ pattern3 300-600msec ^[3-5]\d{5}$
124
+ pattern4 600msec-1sec ^[6-9]\d{5}$
125
+ pattern5 1-2sec ^1\d{6}$
126
+ pattern6 2-10sec ^[2-9]\d{6}$
127
+ pattern7 10sec_over ^\d{8,}$
79
128
  </store>
80
129
  <store>
81
- type forest
82
- subtype datacounter
83
- <template>
84
- unit minute
85
- outcast_unmatched false
86
- aggregate all
87
- tag gf.responsecode.__TAG__
88
- count_key status
89
- pattern1 200 ^200$
90
- pattern2 2xx ^2\d\d$
91
- pattern3 301 ^301$
92
- pattern4 302 ^302$
93
- pattern5 3xx ^3\d\d$
94
- pattern6 403 ^403$
95
- pattern7 404 ^404$
96
- pattern8 410 ^410$
97
- pattern9 4xx ^4\d\d$
98
- pattern10 500 ^5\d\d$
99
- </template>
130
+ @type datacounter
131
+ unit minute
132
+ outcast_unmatched false
133
+ aggregate tag
134
+ tag_prefix gf.responsecode
135
+ output_per_tag yes
136
+ count_key status
137
+ pattern1 200 ^200$
138
+ pattern2 2xx ^2\d\d$
139
+ pattern3 301 ^301$
140
+ pattern4 302 ^302$
141
+ pattern5 3xx ^3\d\d$
142
+ pattern6 403 ^403$
143
+ pattern7 404 ^404$
144
+ pattern8 410 ^410$
145
+ pattern9 4xx ^4\d\d$
146
+ pattern10 500 ^5\d\d$
100
147
  </store>
101
148
  <store>
102
- type forest
103
- subtype datacounter
104
- <template>
105
- unit minute
106
- count_key agent
107
- outcast_unmatched false
108
- aggregate all
109
- tag gf.useragent.__TAG__
110
- pattern1 api HttpRequest
111
- pattern2 robot (spider|bot|crawler|\+http\:)
112
- pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
113
- pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
114
- pattern5 pc .+
115
- </template>
149
+ @type datacounter
150
+ unit minute
151
+ count_key agent
152
+ outcast_unmatched false
153
+ aggregate tag
154
+ tag_prefix gf.useragent
155
+ output_per_tag yes
156
+ pattern1 api HttpRequest
157
+ pattern2 robot (spider|bot|crawler|\+http\:)
158
+ pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
159
+ pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
160
+ pattern5 pc .+
116
161
  </store>
117
162
  </match>
118
163
 
119
164
 
120
165
  # Graph
121
166
  <match gf.responsetime.**>
122
- type forest
123
- subtype growthforecast
167
+ @type growthforecast
124
168
  remove_prefix gf.responsetime.site
125
- <template>
126
- gfapi_url http://localhost:5125/api/
127
- service __TAG__
128
- section response_time
129
- name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
130
- </template>
169
+ gfapi_url http://localhost:5125/api/
170
+ tag_for service
171
+ section response_time
172
+ name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
131
173
  </match>
132
174
 
133
175
  <match gf.responsecode.**>
134
- type forest
135
- subtype growthforecast
176
+ @type growthforecast
136
177
  remove_prefix gf.responsecode.site
137
- <template>
138
- gfapi_url http://localhost:5125/api/
139
- service __TAG__
140
- section response_code
141
- name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
142
- </template>
178
+ gfapi_url http://localhost:5125/api/
179
+ tag_for service
180
+ section response_code
181
+ name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
143
182
  </match>
144
183
 
145
184
  <match gf.useragent.**>
146
- type forest
147
- subtype growthforecast
185
+ @type growthforecast
148
186
  remove_prefix gf.useragent.site
149
- <template>
150
- gfapi_url http://localhost:5125/api/
151
- service __TAG__
152
- section useragent
153
- name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
154
- </template>
187
+ gfapi_url http://localhost:5125/api/
188
+ tag_for service
189
+ section useragent
190
+ name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
155
191
  </match>
156
192
 
data/example2.conf CHANGED
@@ -7,10 +7,13 @@
7
7
  # growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
8
8
 
9
9
  <source>
10
- type tail
10
+ @type tail
11
11
  path /var/log/httpd/access_log
12
- format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
13
- time_format %d/%b/%Y:%H:%M:%S %z
12
+ <parse>
13
+ @type regexp
14
+ expression /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
15
+ time_format %d/%b/%Y:%H:%M:%S %z
16
+ </parse>
14
17
  tag td.apache.access
15
18
  pos_file /var/log/td-agent/apache_access.pos
16
19
  </source>
@@ -18,60 +21,83 @@
18
21
 
19
22
  # Extract specified virtual domain
20
23
  <match td.apache.access>
21
- type rewrite_tag_filter
22
- rewriterule1 domain ^maps\.google\.com$ filter.GoogleMap
24
+ @type rewrite_tag_filter
25
+ <rule>
26
+ key domain
27
+ pattern /^maps\.google\.com$/
28
+ tag filter.GoogleMap
29
+ </rule>
23
30
  </match>
24
31
 
25
32
 
26
33
  # Filtering
27
34
  <match filter.GoogleMap>
28
- type rewrite_tag_filter
29
- rewriterule1 path ^/(img|css|js|static|assets)/ clear
30
- rewriterule2 status ^(?!200)$ clear
31
- rewriterule3 method ^(?!GET)$ clear
32
- rewriterule4 agent (spider|bot|crawler|\+http\:) clear
33
- rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
34
- rewriterule6 path .+ site.GoogleMap.front
35
+ @type rewrite_tag_filter
36
+ <rule>
37
+ key path
38
+ pattern /^\/(img|css|js|static|assets)\//
39
+ tag clear
40
+ </rule>
41
+ <rule>
42
+ key status
43
+ pattern /^(?!200)$/
44
+ tag clear
45
+ </rule>
46
+ <rule>
47
+ key method
48
+ pattern /^(?!GET)$/
49
+ tag clear
50
+ </rule>
51
+ <rule>
52
+ key agent
53
+ pattern /(spider|bot|crawler|\+http\:)/
54
+ tag clean
55
+ </rule>
56
+ <rule>
57
+ key path
58
+ pattern /^\/(admin|api|backend)/
59
+ tag site.GoogleMap.backend
60
+ </rule>
61
+ <rule>
62
+ key path
63
+ pattern /.+/
64
+ tag site.GoogleMap.front
65
+ </rule>
35
66
  </match>
36
67
 
37
68
 
38
69
  # Summarize
39
70
  <match site.**>
40
- type forest
41
- subtype datacounter
42
- <template>
43
- unit minute
44
- count_key response_time
45
- outcast_unmatched false
46
- aggregate all
47
- tag gf.responsetime.__TAG__
48
- pattern1 0-100msec ^\d{1,5}$
49
- pattern2 100-300msec ^[1-2]\d{5}$
50
- pattern3 300-600msec ^[3-5]\d{5}$
51
- pattern4 600msec-1sec ^[6-9]\d{5}$
52
- pattern5 1-2sec ^1\d{6}$
53
- pattern6 2-10sec ^[2-9]\d{6}$
54
- pattern7 10sec_over ^\d{8,}$
55
- </template>
71
+ @type datacounter
72
+ unit minute
73
+ count_key response_time
74
+ outcast_unmatched false
75
+ aggregate tag
76
+ tag_prefix gf.responsetime
77
+ output_per_tag yes
78
+ pattern1 0-100msec ^\d{1,5}$
79
+ pattern2 100-300msec ^[1-2]\d{5}$
80
+ pattern3 300-600msec ^[3-5]\d{5}$
81
+ pattern4 600msec-1sec ^[6-9]\d{5}$
82
+ pattern5 1-2sec ^1\d{6}$
83
+ pattern6 2-10sec ^[2-9]\d{6}$
84
+ pattern7 10sec_over ^\d{8,}$
56
85
  </match>
57
86
 
58
87
 
59
88
  # Graph
60
89
  <match gf.responsetime.**>
61
- type forest
62
- subtype growthforecast
90
+ @type growthforecast
63
91
  remove_prefix gf.responsetime.site
64
- <template>
65
- gfapi_url http://localhost:5125/api/
66
- service __TAG__
67
- section response_time
68
- name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
69
- </template>
92
+ gfapi_url http://localhost:5125/api/
93
+ tag_for service
94
+ section response_time
95
+ name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
70
96
  </match>
71
97
 
72
98
 
73
99
  # Clear tag
74
100
  <match clear>
75
- type null
101
+ @type null
76
102
  </match>
77
103
 
data/fluent-plugin-rewrite-tag-filter.gemspec CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-rewrite-tag-filter"
6
- s.version = "2.0.0"
6
+ s.version = "2.1.1"
7
7
  s.license = "Apache-2.0"
8
8
  s.authors = ["Kentaro Yoshida"]
9
9
  s.email = ["y.ken.studio@gmail.com"]
@@ -17,5 +17,6 @@ Gem::Specification.new do |s|
17
17
 
18
18
  s.add_development_dependency "test-unit", ">= 3.1.0"
19
19
  s.add_development_dependency "rake"
20
- s.add_runtime_dependency "fluentd", "~> 0.14.2"
20
+ s.add_runtime_dependency "fluentd", [">= 0.14.2", "< 2"]
21
+ s.add_runtime_dependency "fluent-config-regexp-type"
21
22
  end
data/lib/fluent/plugin/out_rewrite_tag_filter.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require "fluent/plugin/output"
2
+ require "fluent/config/regexp_type"
2
3
 
3
4
  class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
4
5
  Fluent::Plugin.register_output('rewrite_tag_filter', self)
@@ -16,9 +17,7 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
16
17
  desc "The field name to which the regular expression is applied"
17
18
  config_param :key, :string
18
19
  desc "The regular expression"
19
- config_param :pattern do |value|
20
- Regexp.compile(value)
21
- end
20
+ config_param :pattern, :regexp
22
21
  desc "New tag"
23
22
  config_param :tag, :string
24
23
  desc "If true, rewrite tag when unmatch pattern"
@@ -67,15 +66,18 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
67
66
  end
68
67
 
69
68
  def process(tag, es)
69
+ placeholder = get_placeholder(tag)
70
70
  es.each do |time, record|
71
- rewrited_tag = rewrite_tag(tag, record)
72
- next if rewrited_tag.nil? || tag == rewrited_tag
71
+ rewrited_tag = rewrite_tag(tag, record, placeholder)
72
+ if rewrited_tag.nil? || tag == rewrited_tag
73
+ log.trace("rewrite_tag_filter: tag has not been rewritten", record)
74
+ next
75
+ end
73
76
  router.emit(rewrited_tag, time, record)
74
77
  end
75
78
  end
76
79
 
77
- def rewrite_tag(tag, record)
78
- placeholder = get_placeholder(tag)
80
+ def rewrite_tag(tag, record, placeholder)
79
81
  @rewriterules.each do |record_accessor, regexp, match_operator, rewritetag|
80
82
  rewritevalue = record_accessor.call(record).to_s
81
83
  next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
data/test/plugin/test_out_rewrite_tag_filter.rb CHANGED
@@ -17,6 +17,30 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
17
17
  create_driver(conf)
18
18
  end
19
19
  end
20
+
21
+ test "pattern with character classes" do
22
+ conf = %[
23
+ <rule>
24
+ key $['email']['domain']
25
+ pattern /[sv]d[a-z]+\\d*$/
26
+ tag $2.$1
27
+ </rule>
28
+ ]
29
+ d = create_driver(conf)
30
+ assert_equal(/[sv]d[a-z]+\d*$/, d.instance.rules.first.pattern)
31
+ end
32
+
33
+ test "pattern w/o slashes" do
34
+ conf = %[
35
+ <rule>
36
+ key $['email']['domain']
37
+ pattern .+
38
+ tag $2.$1
39
+ </rule>
40
+ ]
41
+ d = create_driver(conf)
42
+ assert_equal(/.+/, d.instance.rules.first.pattern)
43
+ end
20
44
  end
21
45
 
22
46
  sub_test_case "section style" do
@@ -260,5 +284,27 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
260
284
  events = d.events
261
285
  assert_equal "com.example", events[0][0]
262
286
  end
287
+
288
+ test "tag has not been rewritten and log_level trace" do
289
+ conf = %[
290
+ @log_level trace
291
+ <rule>
292
+ key $['email']['domain']
293
+ pattern ^(example)\.(com)$
294
+ tag $2.$1
295
+ </rule>
296
+ ]
297
+ d = create_driver(conf)
298
+ d.run(default_tag: "input") do
299
+ d.feed({ "email" => { "localpart" => "john", "domain" => "example.com" }})
300
+ d.feed({ "email" => { "localpart" => "doe", "domain" => "example.jp" }})
301
+ end
302
+ events = d.events
303
+ assert_equal(1, events.size)
304
+ log = d.logs.grep(/\[trace\]/).first
305
+ assert_equal('rewrite_tag_filter: tag has not been rewritten email={"localpart"=>"doe", "domain"=>"example.jp"}',
306
+ log.slice(/\[trace\]: (.+)$/, 1))
307
+ assert_equal "com.example", events[0][0]
308
+ end
263
309
  end
264
310
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-rewrite-tag-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kentaro Yoshida
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-11-06 00:00:00.000000000 Z
11
+ date: 2018-11-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: test-unit
@@ -42,16 +42,36 @@ dependencies:
42
42
  name: fluentd
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: 0.14.2
48
+ - - "<"
49
+ - !ruby/object:Gem::Version
50
+ version: '2'
48
51
  type: :runtime
49
52
  prerelease: false
50
53
  version_requirements: !ruby/object:Gem::Requirement
51
54
  requirements:
52
- - - "~>"
55
+ - - ">="
53
56
  - !ruby/object:Gem::Version
54
57
  version: 0.14.2
58
+ - - "<"
59
+ - !ruby/object:Gem::Version
60
+ version: '2'
61
+ - !ruby/object:Gem::Dependency
62
+ name: fluent-config-regexp-type
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ type: :runtime
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
55
75
  description:
56
76
  email:
57
77
  - y.ken.studio@gmail.com
@@ -91,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
91
111
  version: '0'
92
112
  requirements: []
93
113
  rubyforge_project:
94
- rubygems_version: 2.6.13
114
+ rubygems_version: 2.7.6
95
115
  signing_key:
96
116
  specification_version: 4
97
117
  summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.