fluent-plugin-rewrite-tag-filter 2.0.0 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +30 -16
  3. data/example.conf +132 -96
  4. data/example2.conf +63 -37
  5. data/fluent-plugin-rewrite-tag-filter.gemspec +3 -2
  6. data/lib/fluent/plugin/out_rewrite_tag_filter.rb +9 -7
  7. data/test/plugin/test_out_rewrite_tag_filter.rb +46 -0
  8. metadata +25 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 2644562a37644d56fa8712f13d46aa0ff42cb9ec
4
- data.tar.gz: 22ac2b253497f823babee508e41d695b9bdc8218
2
+ SHA256:
3
+ metadata.gz: 9a976b12b3e1ab4b56116a3690d0c48949ff34788f1b165b39909c52d4e1983d
4
+ data.tar.gz: 435ca4ea943d418f2803ee787422271850ededecbe20bd82efa518bb5ff12a7a
5
5
  SHA512:
6
- metadata.gz: eaa2ee232b41ba0b6f387adb69621e78a411919d8d074c3ce014d344b4211a1dad7d47b61a06698cbc4fa0791de55a87111bf91383c47cee3f56f71299ad53e1
7
- data.tar.gz: ff33f9efd2106fc70e85023a139563764aa8414329066459bd9dd177133ee9ff8b9bff20119a50d8621c20573f3d7cdcfc3cb37c1d67a4f6210190b9523bb878
6
+ metadata.gz: 49b4bf86556799b2ac17c37e214754af8ea80489d3817a7026e4aac725ac360605154538e885f4c5cc6843424a1d9575ea383a4fa99c6e3b045169347855ecd2
7
+ data.tar.gz: 7f34dc3e1f1d5178051d3e2af5d433064bb8219638f4181854c6e712341fa0bdbe52f8baa527e9b66967c79686ea2eccf0641fcaadfce22379e47b9bb7747c59
data/README.md CHANGED
@@ -7,6 +7,8 @@ Re-emit the record with rewrited tag when a value matches/unmatches with a regul
7
7
  Also you can change a tag from Apache log by domain, status code (ex. 500 error),
8
8
  user-agent, request-uri, regex-backreference and so on with regular expression.
9
9
 
10
+ This is an output plugin because fluentd's `filter` doesn't allow tag rewrite.
11
+
10
12
  ## Requirements
11
13
 
12
14
  | fluent-plugin-rewrite-tag-filter | Fluentd | Ruby |
@@ -16,23 +18,25 @@ user-agent, request-uri, regex-backreference and so on with regular expression.
16
18
 
17
19
  ## Installation
18
20
 
19
- Install with `gem`, `fluent-gem` or `td-agent-gem` command as:
21
+ Install with `gem` or `td-agent-gem` command as:
20
22
 
21
23
  ```
22
24
  # for system installed fluentd
23
25
  $ gem install fluent-plugin-rewrite-tag-filter
24
26
 
25
- # for td-agent (Legacy)
26
- $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
27
-
28
27
  # for td-agent2 (with fluentd v0.12)
29
- $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
28
+ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.6.0
29
+
30
+ # for td-agent3 (with fluentd v0.14)
31
+ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter
30
32
  ```
31
33
 
34
+ For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/articles/plugin-management)
35
+
32
36
  ## Configuration
33
37
 
34
38
  * **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
35
- * Deprecated: Use <rule> section
39
+ * Obsoleted: Use <rule> section
36
40
  * **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
37
41
  * Default value: no
38
42
  * **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
@@ -42,7 +46,9 @@ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
42
46
  ### \<rule\> section (optional) (multiple)
43
47
 
44
48
  * **key** (string) (required): The field name to which the regular expression is applied
45
- * **pattern** (regexp) (required): The regular expression
49
+ * **pattern** (regexp) (required): The regular expression.
50
+ `/regexp/` is preferred because `/regexp/` style can support character classes such as `/[a-z]/`.
51
+ The pattern without slashes will cause errors if you use patterns start with character classes.
46
52
  * **tag** (string) (required): New tag
47
53
  * **invert** (bool) (optional): If true, rewrite tag when unmatch pattern
48
54
  * Default value: `false`
@@ -70,39 +76,39 @@ It's a sample to exclude some static file log before split tag by domain.
70
76
  capitalize_regex_backreference yes
71
77
  <rule>
72
78
  key path
73
- pattern \.(gif|jpe?g|png|pdf|zip)$
79
+ pattern /\.(gif|jpe?g|png|pdf|zip)$/
74
80
  tag clear
75
81
  </rule>
76
82
  <rule>
77
83
  key status
78
- pattern ^200$
84
+ pattern /^200$/
79
85
  tag clear
80
86
  invert true
81
87
  </rule>
82
88
  <rule>
83
89
  key domain
84
- pattern ^.+\.com$
90
+ pattern /^.+\.com$/
85
91
  tag clear
86
92
  invert true
87
93
  </rule>
88
94
  <rule>
89
95
  key domain
90
- pattern ^maps\.example\.com$
96
+ pattern /^maps\.example\.com$/
91
97
  tag site.ExampleMaps
92
98
  </rule>
93
99
  <rule>
94
100
  key domain
95
- pattern ^news\.example\.com$
101
+ pattern /^news\.example\.com$/
96
102
  tag site.ExampleNews
97
103
  </rule>
98
104
  <rule>
99
105
  key domain
100
- pattern ^(mail)\.(example)\.com$
106
+ pattern /^(mail)\.(example)\.com$/
101
107
  tag site.$2$1
102
108
  </rule>
103
109
  <rule>
104
110
  key domain
105
- pattern .+
111
+ pattern /.+/
106
112
  tag site.unmatched
107
113
  </rule>
108
114
  </match>
@@ -157,7 +163,11 @@ Dot notation:
157
163
  ```
158
164
  <match kubernetes.**>
159
165
  @type rewrite_tag_filter
160
- rewriterule1 $.kubernetes.namespace_name ^(.+)$ $1.${tag}
166
+ <rule>
167
+ key $.kubernetes.namespace_name
168
+ pattern ^(.+)$
169
+ tag $1.${tag}
170
+ </rule>
161
171
  </match>
162
172
  ```
163
173
 
@@ -166,7 +176,11 @@ Bracket notation:
166
176
  ```
167
177
  <match kubernetes.**>
168
178
  @type rewrite_tag_filter
169
- rewriterule1 $['kubernetes']['namespace_name'] ^(.+)$ $1.${tag}
179
+ <rule>
180
+ key $['kubernetes']['namespace_name']
181
+ pattern ^(.+)$
182
+ tag $1.${tag}
183
+ </rule>
170
184
  </match>
171
185
  ```
172
186
 
data/example.conf CHANGED
@@ -2,15 +2,17 @@
2
2
 
3
3
  # appearing plugins:
4
4
  # rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
5
- # forest: http://rubygems.org/gems/fluent-plugin-forest
6
5
  # datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
7
6
  # growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
8
7
 
9
8
  <source>
10
- type tail
9
+ @type tail
11
10
  path /var/log/httpd/access_log
12
- format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
13
- time_format %d/%b/%Y:%H:%M:%S %z
11
+ <parse>
12
+ @type regexp
13
+ expression /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
14
+ time_format %d/%b/%Y:%H:%M:%S %z
15
+ </parse>
14
16
  tag td.apache.access
15
17
  pos_file /var/log/td-agent/apache_access.pos
16
18
  </source>
@@ -18,139 +20,173 @@
18
20
 
19
21
  # Extract specified virtual domain
20
22
  <match td.apache.access>
21
- type copy
23
+ @type copy
22
24
  <store>
23
- type rewrite_tag_filter
25
+ @type rewrite_tag_filter
24
26
  capitalize_regex_backreference yes
25
- rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
27
+ <rule>
28
+ key domain
29
+ pattern /^(maps|news|mail)\.google\.com$/
30
+ tag site.Google$1
31
+ </rule>
26
32
  </store>
27
33
  <store>
28
- type rewrite_tag_filter
34
+ @type rewrite_tag_filter
29
35
  capitalize_regex_backreference yes
30
- rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
36
+ <rule>
37
+ key domain
38
+ pattern /^(maps)\.google\.com$/
39
+ tag sitepath.Google$1
40
+ </rule>
31
41
  </store>
32
42
  </match>
33
43
 
34
44
 
35
45
  # Second level analyzing
36
46
  <match sitepath.GoogleMaps>
47
+ @type copy
37
48
  <store>
38
- type rewrite_tag_filter
39
- rewriterule1 path ^/labs site.GoogleMaps.Labs
40
- rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
41
- rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
49
+ @type rewrite_tag_filter
50
+ <rule>
51
+ key path
52
+ pattern /^\/labs/
53
+ tag site.GoogleMaps.Labs
54
+ </rule>
55
+ <rule>
56
+ key path
57
+ pattern /^\/static/\d+/
58
+ tag site.GoogleMaps.Static
59
+ </rule>
60
+ <rule>
61
+ key path
62
+ pattern /^\/help\/maps\/(streetview|getmaps)/
63
+ tag site.GoogleMaps.Help
64
+ </rule>
42
65
  </store>
43
66
  <store>
44
- type rewrite_tag_filter
45
- rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
46
- rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
67
+ @type rewrite_tag_filter
68
+ <rule>
69
+ key referer
70
+ pattern /headlines\.yahoo\.co\.jp/
71
+ tag site.GoogleMaps.referer_YahooHeadlines
72
+ </rule>
73
+ <rule>
74
+ key referer
75
+ pattern /news\.livedoor\.com/
76
+ tag site.GoogleMaps.referer_LivedoorNews
77
+ </rule>
47
78
  </store>
48
79
  <store>
49
- type rewrite_tag_filter
50
- rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
51
- rewriterule2 agent ^.* iPhone .+Googlebot-Mobile/.*$ site.GoogleMaps.agent_GooglebotSmartphone
52
- rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
53
- rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
54
- rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
80
+ @type rewrite_tag_filter
81
+ <rule>
82
+ key agent
83
+ pattern /Googlebot\//
84
+ tag site.GoogleMaps.agent_Googlebot
85
+ </rule>
86
+ <rule>
87
+ key agent
88
+ pattern /^.* iPhone .+Googlebot-Mobile\/.*$/
89
+ tag site.GoogleMaps.agent_GooglebotSmartphone
90
+ </rule>
91
+ <rule>
92
+ key agent
93
+ pattern /Googlebot-Mobile\//
94
+ tag site.GoogleMaps.agent_GooglebotMobile
95
+ </rule>
96
+ <rule>
97
+ key agent
98
+ pattern /bingbot/
99
+ tag site.GoogleMaps.agent_Bingbot
100
+ </rule>
101
+ <rule>
102
+ key agent
103
+ pattern /Baiduspider/
104
+ tag site.GoogleMaps.agent_Baiduspider
105
+ </rule>
55
106
  </store>
56
107
  </match>
57
108
 
58
109
 
59
110
  # Summarize
60
111
  <match site.**>
61
- type copy
112
+ @type copy
62
113
  <store>
63
- type forest
64
- subtype datacounter
65
- <template>
66
- unit minute
67
- count_key response_time
68
- outcast_unmatched false
69
- aggregate all
70
- tag gf.responsetime.__TAG__
71
- pattern1 0-100msec ^\d{1,5}$
72
- pattern2 100-300msec ^[1-2]\d{5}$
73
- pattern3 300-600msec ^[3-5]\d{5}$
74
- pattern4 600msec-1sec ^[6-9]\d{5}$
75
- pattern5 1-2sec ^1\d{6}$
76
- pattern6 2-10sec ^[2-9]\d{6}$
77
- pattern7 10sec_over ^\d{8,}$
78
- </template>
114
+ @type datacounter
115
+ unit minute
116
+ count_key response_time
117
+ outcast_unmatched false
118
+ aggregate tag
119
+ tag_prefix gf.responsetime
120
+ output_per_tag yes
121
+ pattern1 0-100msec ^\d{1,5}$
122
+ pattern2 100-300msec ^[1-2]\d{5}$
123
+ pattern3 300-600msec ^[3-5]\d{5}$
124
+ pattern4 600msec-1sec ^[6-9]\d{5}$
125
+ pattern5 1-2sec ^1\d{6}$
126
+ pattern6 2-10sec ^[2-9]\d{6}$
127
+ pattern7 10sec_over ^\d{8,}$
79
128
  </store>
80
129
  <store>
81
- type forest
82
- subtype datacounter
83
- <template>
84
- unit minute
85
- outcast_unmatched false
86
- aggregate all
87
- tag gf.responsecode.__TAG__
88
- count_key status
89
- pattern1 200 ^200$
90
- pattern2 2xx ^2\d\d$
91
- pattern3 301 ^301$
92
- pattern4 302 ^302$
93
- pattern5 3xx ^3\d\d$
94
- pattern6 403 ^403$
95
- pattern7 404 ^404$
96
- pattern8 410 ^410$
97
- pattern9 4xx ^4\d\d$
98
- pattern10 500 ^5\d\d$
99
- </template>
130
+ @type datacounter
131
+ unit minute
132
+ outcast_unmatched false
133
+ aggregate tag
134
+ tag_prefix gf.responsecode
135
+ output_per_tag yes
136
+ count_key status
137
+ pattern1 200 ^200$
138
+ pattern2 2xx ^2\d\d$
139
+ pattern3 301 ^301$
140
+ pattern4 302 ^302$
141
+ pattern5 3xx ^3\d\d$
142
+ pattern6 403 ^403$
143
+ pattern7 404 ^404$
144
+ pattern8 410 ^410$
145
+ pattern9 4xx ^4\d\d$
146
+ pattern10 500 ^5\d\d$
100
147
  </store>
101
148
  <store>
102
- type forest
103
- subtype datacounter
104
- <template>
105
- unit minute
106
- count_key agent
107
- outcast_unmatched false
108
- aggregate all
109
- tag gf.useragent.__TAG__
110
- pattern1 api HttpRequest
111
- pattern2 robot (spider|bot|crawler|\+http\:)
112
- pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
113
- pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
114
- pattern5 pc .+
115
- </template>
149
+ @type datacounter
150
+ unit minute
151
+ count_key agent
152
+ outcast_unmatched false
153
+ aggregate tag
154
+ tag_prefix gf.useragent
155
+ output_per_tag yes
156
+ pattern1 api HttpRequest
157
+ pattern2 robot (spider|bot|crawler|\+http\:)
158
+ pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
159
+ pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
160
+ pattern5 pc .+
116
161
  </store>
117
162
  </match>
118
163
 
119
164
 
120
165
  # Graph
121
166
  <match gf.responsetime.**>
122
- type forest
123
- subtype growthforecast
167
+ @type growthforecast
124
168
  remove_prefix gf.responsetime.site
125
- <template>
126
- gfapi_url http://localhost:5125/api/
127
- service __TAG__
128
- section response_time
129
- name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
130
- </template>
169
+ gfapi_url http://localhost:5125/api/
170
+ tag_for service
171
+ section response_time
172
+ name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
131
173
  </match>
132
174
 
133
175
  <match gf.responsecode.**>
134
- type forest
135
- subtype growthforecast
176
+ @type growthforecast
136
177
  remove_prefix gf.responsecode.site
137
- <template>
138
- gfapi_url http://localhost:5125/api/
139
- service __TAG__
140
- section response_code
141
- name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
142
- </template>
178
+ gfapi_url http://localhost:5125/api/
179
+ tag_for service
180
+ section response_code
181
+ name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
143
182
  </match>
144
183
 
145
184
  <match gf.useragent.**>
146
- type forest
147
- subtype growthforecast
185
+ @type growthforecast
148
186
  remove_prefix gf.useragent.site
149
- <template>
150
- gfapi_url http://localhost:5125/api/
151
- service __TAG__
152
- section useragent
153
- name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
154
- </template>
187
+ gfapi_url http://localhost:5125/api/
188
+ tag_for service
189
+ section useragent
190
+ name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
155
191
  </match>
156
192
 
data/example2.conf CHANGED
@@ -7,10 +7,13 @@
7
7
  # growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
8
8
 
9
9
  <source>
10
- type tail
10
+ @type tail
11
11
  path /var/log/httpd/access_log
12
- format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
13
- time_format %d/%b/%Y:%H:%M:%S %z
12
+ <parse>
13
+ @type regexp
14
+ expression /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
15
+ time_format %d/%b/%Y:%H:%M:%S %z
16
+ </parse>
14
17
  tag td.apache.access
15
18
  pos_file /var/log/td-agent/apache_access.pos
16
19
  </source>
@@ -18,60 +21,83 @@
18
21
 
19
22
  # Extract specified virtual domain
20
23
  <match td.apache.access>
21
- type rewrite_tag_filter
22
- rewriterule1 domain ^maps\.google\.com$ filter.GoogleMap
24
+ @type rewrite_tag_filter
25
+ <rule>
26
+ key domain
27
+ pattern /^maps\.google\.com$/
28
+ tag filter.GoogleMap
29
+ </rule>
23
30
  </match>
24
31
 
25
32
 
26
33
  # Filtering
27
34
  <match filter.GoogleMap>
28
- type rewrite_tag_filter
29
- rewriterule1 path ^/(img|css|js|static|assets)/ clear
30
- rewriterule2 status ^(?!200)$ clear
31
- rewriterule3 method ^(?!GET)$ clear
32
- rewriterule4 agent (spider|bot|crawler|\+http\:) clear
33
- rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
34
- rewriterule6 path .+ site.GoogleMap.front
35
+ @type rewrite_tag_filter
36
+ <rule>
37
+ key path
38
+ pattern /^\/(img|css|js|static|assets)\//
39
+ tag clear
40
+ </rule>
41
+ <rule>
42
+ key status
43
+ pattern /^(?!200)$/
44
+ tag clear
45
+ </rule>
46
+ <rule>
47
+ key method
48
+ pattern /^(?!GET)$/
49
+ tag clear
50
+ </rule>
51
+ <rule>
52
+ key agent
53
+ pattern /(spider|bot|crawler|\+http\:)/
54
+ tag clean
55
+ </rule>
56
+ <rule>
57
+ key path
58
+ pattern /^\/(admin|api|backend)/
59
+ tag site.GoogleMap.backend
60
+ </rule>
61
+ <rule>
62
+ key path
63
+ pattern /.+/
64
+ tag site.GoogleMap.front
65
+ </rule>
35
66
  </match>
36
67
 
37
68
 
38
69
  # Summarize
39
70
  <match site.**>
40
- type forest
41
- subtype datacounter
42
- <template>
43
- unit minute
44
- count_key response_time
45
- outcast_unmatched false
46
- aggregate all
47
- tag gf.responsetime.__TAG__
48
- pattern1 0-100msec ^\d{1,5}$
49
- pattern2 100-300msec ^[1-2]\d{5}$
50
- pattern3 300-600msec ^[3-5]\d{5}$
51
- pattern4 600msec-1sec ^[6-9]\d{5}$
52
- pattern5 1-2sec ^1\d{6}$
53
- pattern6 2-10sec ^[2-9]\d{6}$
54
- pattern7 10sec_over ^\d{8,}$
55
- </template>
71
+ @type datacounter
72
+ unit minute
73
+ count_key response_time
74
+ outcast_unmatched false
75
+ aggregate tag
76
+ tag_prefix gf.responsetime
77
+ output_per_tag yes
78
+ pattern1 0-100msec ^\d{1,5}$
79
+ pattern2 100-300msec ^[1-2]\d{5}$
80
+ pattern3 300-600msec ^[3-5]\d{5}$
81
+ pattern4 600msec-1sec ^[6-9]\d{5}$
82
+ pattern5 1-2sec ^1\d{6}$
83
+ pattern6 2-10sec ^[2-9]\d{6}$
84
+ pattern7 10sec_over ^\d{8,}$
56
85
  </match>
57
86
 
58
87
 
59
88
  # Graph
60
89
  <match gf.responsetime.**>
61
- type forest
62
- subtype growthforecast
90
+ @type growthforecast
63
91
  remove_prefix gf.responsetime.site
64
- <template>
65
- gfapi_url http://localhost:5125/api/
66
- service __TAG__
67
- section response_time
68
- name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
69
- </template>
92
+ gfapi_url http://localhost:5125/api/
93
+ tag_for service
94
+ section response_time
95
+ name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
70
96
  </match>
71
97
 
72
98
 
73
99
  # Clear tag
74
100
  <match clear>
75
- type null
101
+ @type null
76
102
  </match>
77
103
 
data/fluent-plugin-rewrite-tag-filter.gemspec CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-rewrite-tag-filter"
6
- s.version = "2.0.0"
6
+ s.version = "2.1.1"
7
7
  s.license = "Apache-2.0"
8
8
  s.authors = ["Kentaro Yoshida"]
9
9
  s.email = ["y.ken.studio@gmail.com"]
@@ -17,5 +17,6 @@ Gem::Specification.new do |s|
17
17
 
18
18
  s.add_development_dependency "test-unit", ">= 3.1.0"
19
19
  s.add_development_dependency "rake"
20
- s.add_runtime_dependency "fluentd", "~> 0.14.2"
20
+ s.add_runtime_dependency "fluentd", [">= 0.14.2", "< 2"]
21
+ s.add_runtime_dependency "fluent-config-regexp-type"
21
22
  end
data/lib/fluent/plugin/out_rewrite_tag_filter.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require "fluent/plugin/output"
2
+ require "fluent/config/regexp_type"
2
3
 
3
4
  class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
4
5
  Fluent::Plugin.register_output('rewrite_tag_filter', self)
@@ -16,9 +17,7 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
16
17
  desc "The field name to which the regular expression is applied"
17
18
  config_param :key, :string
18
19
  desc "The regular expression"
19
- config_param :pattern do |value|
20
- Regexp.compile(value)
21
- end
20
+ config_param :pattern, :regexp
22
21
  desc "New tag"
23
22
  config_param :tag, :string
24
23
  desc "If true, rewrite tag when unmatch pattern"
@@ -67,15 +66,18 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
67
66
  end
68
67
 
69
68
  def process(tag, es)
69
+ placeholder = get_placeholder(tag)
70
70
  es.each do |time, record|
71
- rewrited_tag = rewrite_tag(tag, record)
72
- next if rewrited_tag.nil? || tag == rewrited_tag
71
+ rewrited_tag = rewrite_tag(tag, record, placeholder)
72
+ if rewrited_tag.nil? || tag == rewrited_tag
73
+ log.trace("rewrite_tag_filter: tag has not been rewritten", record)
74
+ next
75
+ end
73
76
  router.emit(rewrited_tag, time, record)
74
77
  end
75
78
  end
76
79
 
77
- def rewrite_tag(tag, record)
78
- placeholder = get_placeholder(tag)
80
+ def rewrite_tag(tag, record, placeholder)
79
81
  @rewriterules.each do |record_accessor, regexp, match_operator, rewritetag|
80
82
  rewritevalue = record_accessor.call(record).to_s
81
83
  next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
data/test/plugin/test_out_rewrite_tag_filter.rb CHANGED
@@ -17,6 +17,30 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
17
17
  create_driver(conf)
18
18
  end
19
19
  end
20
+
21
+ test "pattern with character classes" do
22
+ conf = %[
23
+ <rule>
24
+ key $['email']['domain']
25
+ pattern /[sv]d[a-z]+\\d*$/
26
+ tag $2.$1
27
+ </rule>
28
+ ]
29
+ d = create_driver(conf)
30
+ assert_equal(/[sv]d[a-z]+\d*$/, d.instance.rules.first.pattern)
31
+ end
32
+
33
+ test "pattern w/o slashes" do
34
+ conf = %[
35
+ <rule>
36
+ key $['email']['domain']
37
+ pattern .+
38
+ tag $2.$1
39
+ </rule>
40
+ ]
41
+ d = create_driver(conf)
42
+ assert_equal(/.+/, d.instance.rules.first.pattern)
43
+ end
20
44
  end
21
45
 
22
46
  sub_test_case "section style" do
@@ -260,5 +284,27 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
260
284
  events = d.events
261
285
  assert_equal "com.example", events[0][0]
262
286
  end
287
+
288
+ test "tag has not been rewritten and log_level trace" do
289
+ conf = %[
290
+ @log_level trace
291
+ <rule>
292
+ key $['email']['domain']
293
+ pattern ^(example)\.(com)$
294
+ tag $2.$1
295
+ </rule>
296
+ ]
297
+ d = create_driver(conf)
298
+ d.run(default_tag: "input") do
299
+ d.feed({ "email" => { "localpart" => "john", "domain" => "example.com" }})
300
+ d.feed({ "email" => { "localpart" => "doe", "domain" => "example.jp" }})
301
+ end
302
+ events = d.events
303
+ assert_equal(1, events.size)
304
+ log = d.logs.grep(/\[trace\]/).first
305
+ assert_equal('rewrite_tag_filter: tag has not been rewritten email={"localpart"=>"doe", "domain"=>"example.jp"}',
306
+ log.slice(/\[trace\]: (.+)$/, 1))
307
+ assert_equal "com.example", events[0][0]
308
+ end
263
309
  end
264
310
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-rewrite-tag-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kentaro Yoshida
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-11-06 00:00:00.000000000 Z
11
+ date: 2018-11-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: test-unit
@@ -42,16 +42,36 @@ dependencies:
42
42
  name: fluentd
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: 0.14.2
48
+ - - "<"
49
+ - !ruby/object:Gem::Version
50
+ version: '2'
48
51
  type: :runtime
49
52
  prerelease: false
50
53
  version_requirements: !ruby/object:Gem::Requirement
51
54
  requirements:
52
- - - "~>"
55
+ - - ">="
53
56
  - !ruby/object:Gem::Version
54
57
  version: 0.14.2
58
+ - - "<"
59
+ - !ruby/object:Gem::Version
60
+ version: '2'
61
+ - !ruby/object:Gem::Dependency
62
+ name: fluent-config-regexp-type
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ type: :runtime
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
55
75
  description:
56
76
  email:
57
77
  - y.ken.studio@gmail.com
@@ -91,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
91
111
  version: '0'
92
112
  requirements: []
93
113
  rubyforge_project:
94
- rubygems_version: 2.6.13
114
+ rubygems_version: 2.7.6
95
115
  signing_key:
96
116
  specification_version: 4
97
117
  summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.