fluent-plugin-rewrite-tag-filter 1.6.0 → 2.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +0 -2
- data/README.md +39 -105
- data/Rakefile +1 -2
- data/example.conf +26 -74
- data/example2.conf +13 -41
- data/fluent-plugin-rewrite-tag-filter.gemspec +2 -2
- data/lib/fluent/plugin/out_rewrite_tag_filter.rb +14 -41
- data/test/helper.rb +1 -0
- data/test/plugin/test_out_rewrite_tag_filter.rb +124 -337
- metadata +8 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 47add5af0b04bb562a521772ee1a3fac2872a2f9
|
4
|
+
data.tar.gz: b9ec90c1ef4cca20a7fa70a95c105f74a1a0753a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 84132912d4463769f217023558fc4d522a3bdd0e3aa66ae3a92f1aab63b99c8298a47699913dcdcea27fe56db79ef3d692860ed32b43b969783af10b926d7093
|
7
|
+
data.tar.gz: bc956b79041dc4966d46471425e9b879ae2df2f77f73ac212bd2b467e624ea565d8e259fb6283966018d32422b6bbd92caa504216c50600709f8ef2cff9ff03a
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -7,6 +7,15 @@ Re-emit the record with rewrited tag when a value matches/unmatches with a regul
|
|
7
7
|
Also you can change a tag from Apache log by domain, status code (ex. 500 error),
|
8
8
|
user-agent, request-uri, regex-backreference and so on with regular expression.
|
9
9
|
|
10
|
+
## Requirements
|
11
|
+
|
12
|
+
| fluent-plugin-rewrite-tag-filter | Fluentd | Ruby |
|
13
|
+
|----------------------------------|------------|--------|
|
14
|
+
| >= 2.0.0 | >= v0.14.2 | >= 2.1 |
|
15
|
+
| < 2.0.0 | >= v0.12.0 | >= 1.9 |
|
16
|
+
|
17
|
+
NOTE: fluent-plugin-rewrite-tag-filter v2.0.0 is now RC. We will release stable v2.0.0 soon.
|
18
|
+
|
10
19
|
## Installation
|
11
20
|
|
12
21
|
Install with `gem`, `fluent-gem` or `td-agent-gem` command as:
|
@@ -24,20 +33,24 @@ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
|
|
24
33
|
|
25
34
|
## Configuration
|
26
35
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
36
|
+
### Syntax
|
37
|
+
|
38
|
+
```
|
39
|
+
rewriterule<num> <attribute> <regex_pattern> <new_tag>
|
40
|
+
|
41
|
+
# Optional: Capitalize letter for every matched regex backreference. (ex: maps -> Maps)
|
42
|
+
# for more details, see usage.
|
43
|
+
capitalize_regex_backreference <yes/no> (default no)
|
44
|
+
|
45
|
+
# Optional: remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
|
46
|
+
remove_tag_prefix <string>
|
34
47
|
|
35
|
-
|
48
|
+
# Optional: override hostname command for placeholder. (see the section of "Tag placeholder")
|
49
|
+
hostname_command <string>
|
36
50
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
* **invert** (bool) (optional): If true, rewrite tag when unmatched pattern
|
51
|
+
# Optional: Set log level for this plugin. (ex: trace, debug, info, warn, error, fatal)
|
52
|
+
log_level <string> (default info)
|
53
|
+
```
|
41
54
|
|
42
55
|
### Usage
|
43
56
|
|
@@ -54,49 +67,19 @@ It's a sample to exclude some static file log before split tag by domain.
|
|
54
67
|
</source>
|
55
68
|
|
56
69
|
# "capitalize_regex_backreference yes" affects converting every matched first letter of backreference to upper case. ex: maps -> Maps
|
57
|
-
# At
|
58
|
-
# At
|
59
|
-
# At
|
70
|
+
# At rewriterule2, redirect to tag named "clear" which unmatched for status code 200.
|
71
|
+
# At rewriterule3, redirect to tag named "clear" which is not end with ".com"
|
72
|
+
# At rewriterule6, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
|
60
73
|
<match td.apache.access>
|
61
74
|
@type rewrite_tag_filter
|
62
75
|
capitalize_regex_backreference yes
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
pattern ^200$
|
71
|
-
tag clear
|
72
|
-
invert true
|
73
|
-
</rule>
|
74
|
-
<rule>
|
75
|
-
key domain
|
76
|
-
pattern ^.+\.com$
|
77
|
-
tag clear
|
78
|
-
invert true
|
79
|
-
</rule>
|
80
|
-
<rule>
|
81
|
-
key domain
|
82
|
-
pattern ^maps\.example\.com$
|
83
|
-
tag site.ExampleMaps
|
84
|
-
</rule>
|
85
|
-
<rule>
|
86
|
-
key domain
|
87
|
-
pattern ^news\.example\.com$
|
88
|
-
tag site.ExampleNews
|
89
|
-
</rule>
|
90
|
-
<rule>
|
91
|
-
key domain
|
92
|
-
pattern ^(mail)\.(example)\.com$
|
93
|
-
tag site.$2$1
|
94
|
-
</rule>
|
95
|
-
<rule>
|
96
|
-
key domain
|
97
|
-
pattern .+
|
98
|
-
tag site.unmatched
|
99
|
-
</rule>
|
76
|
+
rewriterule1 path \.(gif|jpe?g|png|pdf|zip)$ clear
|
77
|
+
rewriterule2 status !^200$ clear
|
78
|
+
rewriterule3 domain !^.+\.com$ clear
|
79
|
+
rewriterule4 domain ^maps\.example\.com$ site.ExampleMaps
|
80
|
+
rewriterule5 domain ^news\.example\.com$ site.ExampleNews
|
81
|
+
rewriterule6 domain ^(mail)\.(example)\.com$ site.$2$1
|
82
|
+
rewriterule7 domain .+ site.unmatched
|
100
83
|
</match>
|
101
84
|
|
102
85
|
<match site.*>
|
@@ -142,39 +125,6 @@ $ tailf /var/log/td-agent/td-agent.log
|
|
142
125
|
2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [5, "domain", /.+/, "site.unmatched"]
|
143
126
|
```
|
144
127
|
|
145
|
-
### Nested attributes
|
146
|
-
|
147
|
-
You can handle nested attributes using [filter_record_transformer](https://docs.fluentd.org/v0.14/articles/filter_record_transformer).
|
148
|
-
|
149
|
-
```
|
150
|
-
<filter kubernetes.**>
|
151
|
-
@type kubernetes_metadata
|
152
|
-
</filter>
|
153
|
-
<filter kubernetes.**>
|
154
|
-
@type record_transformer
|
155
|
-
enable_ruby
|
156
|
-
<record>
|
157
|
-
kubernetes_namespace ${record["kubernetes"]["namespace"]}
|
158
|
-
</record>
|
159
|
-
</filter>
|
160
|
-
<match kubernetes.**>
|
161
|
-
@type rewrite_tag_filter
|
162
|
-
rewriterule1 kubernetes_namespace (.+) $1.${tag}
|
163
|
-
</match>
|
164
|
-
```
|
165
|
-
|
166
|
-
```
|
167
|
-
{
|
168
|
-
"kubernetes": {
|
169
|
-
"namespace": "default"
|
170
|
-
}
|
171
|
-
}
|
172
|
-
```
|
173
|
-
|
174
|
-
When original tag is `kubernetes.var.log`, this will be converted to `default.kubernetes.var.log`
|
175
|
-
|
176
|
-
See also #13.
|
177
|
-
|
178
128
|
### Tag placeholder
|
179
129
|
|
180
130
|
It is supported these placeholder for new_tag (rewrited tag).
|
@@ -211,43 +161,27 @@ It's a sample to rewrite a tag with placeholder.
|
|
211
161
|
# It will get "rewrited.access.ExampleMail"
|
212
162
|
<match apache.access>
|
213
163
|
@type rewrite_tag_filter
|
164
|
+
rewriterule1 domain ^(mail)\.(example)\.com$ rewrited.${tag}.$2$1
|
214
165
|
remove_tag_prefix apache
|
215
|
-
<rule>
|
216
|
-
key domain
|
217
|
-
pattern ^(mail)\.(example)\.com$
|
218
|
-
tag rewrited.${tag}.$2$1
|
219
|
-
</rule>
|
220
166
|
</match>
|
221
167
|
|
222
168
|
# It will get "rewrited.ExampleMail.app30-124.foo.com" when hostname is "app30-124.foo.com"
|
223
169
|
<match apache.access>
|
224
170
|
@type rewrite_tag_filter
|
225
|
-
|
226
|
-
key domain
|
227
|
-
pattern ^(mail)\.(example)\.com$
|
228
|
-
tag rewrited.$2$1.${hostname}
|
229
|
-
</rule>
|
171
|
+
rewriterule1 domain ^(mail)\.(example)\.com$ rewrited.$2$1.${hostname}
|
230
172
|
</match>
|
231
173
|
|
232
174
|
# It will get "rewrited.ExampleMail.app30-124" when hostname is "app30-124.foo.com"
|
233
175
|
<match apache.access>
|
234
176
|
@type rewrite_tag_filter
|
177
|
+
rewriterule1 domain ^(mail)\.(example)\.com$ rewrited.$2$1.${hostname}
|
235
178
|
hostname_command hostname -s
|
236
|
-
<rule>
|
237
|
-
key domain
|
238
|
-
pattern ^(mail)\.(example)\.com$
|
239
|
-
tag rewrited.$2$1.${hostname}
|
240
|
-
</rule>
|
241
179
|
</match>
|
242
180
|
|
243
181
|
# It will get "rewrited.game.pool"
|
244
182
|
<match app.game.pool.activity>
|
245
183
|
@type rewrite_tag_filter
|
246
|
-
|
247
|
-
key domain
|
248
|
-
pattern ^.+$
|
249
|
-
tag rewrited.${tag_parts[1]}.${tag_parts[2]}
|
250
|
-
</rule>
|
184
|
+
rewriterule1 domain ^.+$ rewrited.${tag_parts[1]}.${tag_parts[2]}
|
251
185
|
</match>
|
252
186
|
```
|
253
187
|
|
data/Rakefile
CHANGED
data/example.conf
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
# growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
|
8
8
|
|
9
9
|
<source>
|
10
|
-
|
10
|
+
type tail
|
11
11
|
path /var/log/httpd/access_log
|
12
12
|
format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
|
13
13
|
time_format %d/%b/%Y:%H:%M:%S %z
|
@@ -18,24 +18,16 @@
|
|
18
18
|
|
19
19
|
# Extract specified virtual domain
|
20
20
|
<match td.apache.access>
|
21
|
-
|
21
|
+
type copy
|
22
22
|
<store>
|
23
|
-
|
23
|
+
type rewrite_tag_filter
|
24
24
|
capitalize_regex_backreference yes
|
25
|
-
|
26
|
-
key domain
|
27
|
-
pattern ^(maps|news|mail)\.google\.com$
|
28
|
-
tag site.Google$1
|
29
|
-
</rule>
|
25
|
+
rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
|
30
26
|
</store>
|
31
27
|
<store>
|
32
|
-
|
28
|
+
type rewrite_tag_filter
|
33
29
|
capitalize_regex_backreference yes
|
34
|
-
|
35
|
-
key domain
|
36
|
-
pattern ^(maps)\.google\.com$
|
37
|
-
tag sitepath.Google$1
|
38
|
-
</rule>
|
30
|
+
rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
|
39
31
|
</store>
|
40
32
|
</match>
|
41
33
|
|
@@ -43,72 +35,32 @@
|
|
43
35
|
# Second level analyzing
|
44
36
|
<match sitepath.GoogleMaps>
|
45
37
|
<store>
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
tag site.GoogleMaps.Labs
|
51
|
-
</rule>
|
52
|
-
<rule>
|
53
|
-
key path
|
54
|
-
pattern ^/static/\d+
|
55
|
-
tag site.GoogleMaps.Static
|
56
|
-
</rule>
|
57
|
-
<rule>
|
58
|
-
key path
|
59
|
-
pattern ^/help/maps/(streetview|getmaps)
|
60
|
-
tag site.GoogleMaps.Help
|
61
|
-
</rule>
|
38
|
+
type rewrite_tag_filter
|
39
|
+
rewriterule1 path ^/labs site.GoogleMaps.Labs
|
40
|
+
rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
|
41
|
+
rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
|
62
42
|
</store>
|
63
43
|
<store>
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
pattern headlines\.yahoo\.co\.jp
|
68
|
-
tag site.GoogleMaps.referer_YahooHeadlines
|
69
|
-
</rule>
|
70
|
-
<rule>
|
71
|
-
key referer
|
72
|
-
pattern news\.livedoor\.com
|
73
|
-
tag site.GoogleMaps.referer_LivedoorNews
|
74
|
-
</rule>
|
44
|
+
type rewrite_tag_filter
|
45
|
+
rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
|
46
|
+
rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
|
75
47
|
</store>
|
76
48
|
<store>
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
<rule>
|
84
|
-
key agent
|
85
|
-
pattern ^.* iPhone .+Googlebot-Mobile/.*$
|
86
|
-
tag site.GoogleMaps.agent_GooglebotSmartphone
|
87
|
-
</rule>
|
88
|
-
<rule>
|
89
|
-
key agent
|
90
|
-
pattern Googlebot-Mobile/
|
91
|
-
tag site.GoogleMaps.agent_GooglebotMobile
|
92
|
-
</rule>
|
93
|
-
<rule>
|
94
|
-
key agent
|
95
|
-
pattern bingbot
|
96
|
-
tag site.GoogleMaps.agent_Bingbot
|
97
|
-
</rule>
|
98
|
-
<rule>
|
99
|
-
key agent
|
100
|
-
pattern Baiduspider
|
101
|
-
tag site.GoogleMaps.agent_Baiduspider
|
102
|
-
</rule>
|
49
|
+
type rewrite_tag_filter
|
50
|
+
rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
|
51
|
+
rewriterule2 agent ^.* iPhone .+Googlebot-Mobile/.*$ site.GoogleMaps.agent_GooglebotSmartphone
|
52
|
+
rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
|
53
|
+
rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
|
54
|
+
rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
|
103
55
|
</store>
|
104
56
|
</match>
|
105
57
|
|
106
58
|
|
107
59
|
# Summarize
|
108
60
|
<match site.**>
|
109
|
-
|
61
|
+
type copy
|
110
62
|
<store>
|
111
|
-
|
63
|
+
type forest
|
112
64
|
subtype datacounter
|
113
65
|
<template>
|
114
66
|
unit minute
|
@@ -126,7 +78,7 @@
|
|
126
78
|
</template>
|
127
79
|
</store>
|
128
80
|
<store>
|
129
|
-
|
81
|
+
type forest
|
130
82
|
subtype datacounter
|
131
83
|
<template>
|
132
84
|
unit minute
|
@@ -147,7 +99,7 @@
|
|
147
99
|
</template>
|
148
100
|
</store>
|
149
101
|
<store>
|
150
|
-
|
102
|
+
type forest
|
151
103
|
subtype datacounter
|
152
104
|
<template>
|
153
105
|
unit minute
|
@@ -167,7 +119,7 @@
|
|
167
119
|
|
168
120
|
# Graph
|
169
121
|
<match gf.responsetime.**>
|
170
|
-
|
122
|
+
type forest
|
171
123
|
subtype growthforecast
|
172
124
|
remove_prefix gf.responsetime.site
|
173
125
|
<template>
|
@@ -179,7 +131,7 @@
|
|
179
131
|
</match>
|
180
132
|
|
181
133
|
<match gf.responsecode.**>
|
182
|
-
|
134
|
+
type forest
|
183
135
|
subtype growthforecast
|
184
136
|
remove_prefix gf.responsecode.site
|
185
137
|
<template>
|
@@ -191,7 +143,7 @@
|
|
191
143
|
</match>
|
192
144
|
|
193
145
|
<match gf.useragent.**>
|
194
|
-
|
146
|
+
type forest
|
195
147
|
subtype growthforecast
|
196
148
|
remove_prefix gf.useragent.site
|
197
149
|
<template>
|
data/example2.conf
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
# growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
|
8
8
|
|
9
9
|
<source>
|
10
|
-
|
10
|
+
type tail
|
11
11
|
path /var/log/httpd/access_log
|
12
12
|
format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
|
13
13
|
time_format %d/%b/%Y:%H:%M:%S %z
|
@@ -18,54 +18,26 @@
|
|
18
18
|
|
19
19
|
# Extract specified virtual domain
|
20
20
|
<match td.apache.access>
|
21
|
-
|
22
|
-
|
23
|
-
key domain
|
24
|
-
pattern ^maps\.google\.com$
|
25
|
-
tag filter.GoogleMap
|
26
|
-
</rule>
|
21
|
+
type rewrite_tag_filter
|
22
|
+
rewriterule1 domain ^maps\.google\.com$ filter.GoogleMap
|
27
23
|
</match>
|
28
24
|
|
29
25
|
|
30
26
|
# Filtering
|
31
27
|
<match filter.GoogleMap>
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
key status
|
40
|
-
pattern ^(?!200)$
|
41
|
-
tag clear
|
42
|
-
</rule>
|
43
|
-
<rule>
|
44
|
-
key method
|
45
|
-
pattern ^(?!GET)$
|
46
|
-
tag clear
|
47
|
-
</rule>
|
48
|
-
<rule>
|
49
|
-
key agent
|
50
|
-
pattern (spider|bot|crawler|\+http\:)
|
51
|
-
tag clear
|
52
|
-
</rule>
|
53
|
-
<rule>
|
54
|
-
key path
|
55
|
-
pattern ^/(admin|api|backend)
|
56
|
-
tag site.GoogleMap.backend
|
57
|
-
</rule>
|
58
|
-
<rule>
|
59
|
-
key path
|
60
|
-
pattern .+
|
61
|
-
tag site.GoogleMap.front
|
62
|
-
</rule>
|
28
|
+
type rewrite_tag_filter
|
29
|
+
rewriterule1 path ^/(img|css|js|static|assets)/ clear
|
30
|
+
rewriterule2 status ^(?!200)$ clear
|
31
|
+
rewriterule3 method ^(?!GET)$ clear
|
32
|
+
rewriterule4 agent (spider|bot|crawler|\+http\:) clear
|
33
|
+
rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
|
34
|
+
rewriterule6 path .+ site.GoogleMap.front
|
63
35
|
</match>
|
64
36
|
|
65
37
|
|
66
38
|
# Summarize
|
67
39
|
<match site.**>
|
68
|
-
|
40
|
+
type forest
|
69
41
|
subtype datacounter
|
70
42
|
<template>
|
71
43
|
unit minute
|
@@ -86,7 +58,7 @@
|
|
86
58
|
|
87
59
|
# Graph
|
88
60
|
<match gf.responsetime.**>
|
89
|
-
|
61
|
+
type forest
|
90
62
|
subtype growthforecast
|
91
63
|
remove_prefix gf.responsetime.site
|
92
64
|
<template>
|
@@ -100,6 +72,6 @@
|
|
100
72
|
|
101
73
|
# Clear tag
|
102
74
|
<match clear>
|
103
|
-
|
75
|
+
type null
|
104
76
|
</match>
|
105
77
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-rewrite-tag-filter"
|
6
|
-
s.version = "
|
6
|
+
s.version = "2.0.0.rc1"
|
7
7
|
s.license = "Apache-2.0"
|
8
8
|
s.authors = ["Kentaro Yoshida"]
|
9
9
|
s.email = ["y.ken.studio@gmail.com"]
|
@@ -17,5 +17,5 @@ Gem::Specification.new do |s|
|
|
17
17
|
|
18
18
|
s.add_development_dependency "test-unit", ">= 3.1.0"
|
19
19
|
s.add_development_dependency "rake"
|
20
|
-
s.add_runtime_dependency "fluentd",
|
20
|
+
s.add_runtime_dependency "fluentd", "~> 0.14.2"
|
21
21
|
end
|
@@ -1,13 +1,9 @@
|
|
1
|
-
|
1
|
+
require "fluent/plugin/output"
|
2
|
+
|
3
|
+
class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
2
4
|
Fluent::Plugin.register_output('rewrite_tag_filter', self)
|
3
5
|
|
4
|
-
|
5
|
-
class << self
|
6
|
-
unless method_defined?(:desc)
|
7
|
-
def desc(description)
|
8
|
-
end
|
9
|
-
end
|
10
|
-
end
|
6
|
+
helpers :event_emitter
|
11
7
|
|
12
8
|
desc 'Capitalize letter for every matched regex backreference.'
|
13
9
|
config_param :capitalize_regex_backreference, :bool, :default => false
|
@@ -16,19 +12,6 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
16
12
|
desc 'Override hostname command for placeholder.'
|
17
13
|
config_param :hostname_command, :string, :default => 'hostname'
|
18
14
|
|
19
|
-
config_section :rule, :param_name => :rules, :multi => true do
|
20
|
-
desc "The field name to which the regular expression is applied"
|
21
|
-
config_param :key, :string
|
22
|
-
desc "The regular expression"
|
23
|
-
config_param :pattern do |value|
|
24
|
-
Regexp.compile(value)
|
25
|
-
end
|
26
|
-
desc "New tag"
|
27
|
-
config_param :tag, :string
|
28
|
-
desc "If true, rewrite tag when unmatched pattern"
|
29
|
-
config_param :invert, :bool, :default => false
|
30
|
-
end
|
31
|
-
|
32
15
|
MATCH_OPERATOR_EXCLUDE = '!'
|
33
16
|
|
34
17
|
def initialize
|
@@ -43,20 +26,6 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
43
26
|
rewriterule_names = []
|
44
27
|
@hostname = `#{@hostname_command}`.chomp
|
45
28
|
|
46
|
-
@rules.each do |rule|
|
47
|
-
unless rule.tag.match(/\$\{tag_parts\[\d\.\.\.?\d\]\}/).nil? or rule.tag.match(/__TAG_PARTS\[\d\.\.\.?\d\]__/).nil?
|
48
|
-
raise Fluent::ConfigError, "${tag_parts[n]} and __TAG_PARTS[n]__ placeholder does not support range specify at #{rule}"
|
49
|
-
end
|
50
|
-
invert = rule.invert ? MATCH_OPERATOR_EXCLUDE : ""
|
51
|
-
@rewriterules.push([rule.key, rule.pattern, invert, rule.tag])
|
52
|
-
rewriterule_names.push(rule.key + invert + rule.pattern.to_s)
|
53
|
-
end
|
54
|
-
|
55
|
-
deprecated_rewriterule = conf.keys.detect {|k| k =~ /^rewriterule(\d+)$/ }
|
56
|
-
if deprecated_rewriterule
|
57
|
-
k = deprecated_rewriterule.split(" ").first
|
58
|
-
log.warn "rewrite_tag_filter: [DEPRECATED] Use <rule> section instead of #{k}"
|
59
|
-
end
|
60
29
|
conf.keys.select{|k| k =~ /^rewriterule(\d+)$/}.sort_by{|i| i.sub('rewriterule', '').to_i}.each do |key|
|
61
30
|
rewritekey,regexp,rewritetag = parse_rewriterule(conf[key])
|
62
31
|
if regexp.nil? || rewritetag.nil?
|
@@ -85,14 +54,16 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
85
54
|
end
|
86
55
|
end
|
87
56
|
|
88
|
-
def
|
89
|
-
|
57
|
+
def multi_workers_ready?
|
58
|
+
true
|
59
|
+
end
|
60
|
+
|
61
|
+
def process(tag, es)
|
62
|
+
es.each do |time, record|
|
90
63
|
rewrited_tag = rewrite_tag(tag, record)
|
91
64
|
next if rewrited_tag.nil? || tag == rewrited_tag
|
92
65
|
router.emit(rewrited_tag, time, record)
|
93
66
|
end
|
94
|
-
|
95
|
-
chain.next
|
96
67
|
end
|
97
68
|
|
98
69
|
def rewrite_tag(tag, record)
|
@@ -119,9 +90,11 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
119
90
|
end
|
120
91
|
|
121
92
|
def regexp_last_match(regexp, rewritevalue)
|
122
|
-
|
93
|
+
begin
|
94
|
+
return if regexp.nil?
|
123
95
|
regexp.match(rewritevalue)
|
124
|
-
|
96
|
+
rescue ArgumentError => e
|
97
|
+
raise e unless e.message.index('invalid byte sequence in') == 0
|
125
98
|
regexp.match(rewritevalue.scrub('?'))
|
126
99
|
end
|
127
100
|
end
|
data/test/helper.rb
CHANGED
@@ -12,6 +12,7 @@ require 'test/unit'
|
|
12
12
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
13
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
14
|
require 'fluent/test'
|
15
|
+
require 'fluent/test/driver/output'
|
15
16
|
unless ENV.has_key?('VERBOSE')
|
16
17
|
nulllogger = Object.new
|
17
18
|
nulllogger.instance_eval {|obj|
|
@@ -68,20 +68,19 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
68
68
|
rewriterule1 client_name (.+) app.$1
|
69
69
|
]
|
70
70
|
|
71
|
-
def create_driver(conf
|
72
|
-
Fluent::Test::
|
71
|
+
def create_driver(conf = CONFIG)
|
72
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::RewriteTagFilterOutput).configure(conf)
|
73
73
|
end
|
74
74
|
|
75
75
|
sub_test_case "configure" do
|
76
76
|
data("empty" => "",
|
77
77
|
"missing regexp" => "rewriterule1 foo",
|
78
|
-
"
|
79
|
-
"not
|
80
|
-
"not regext 2" => "rewriterule1 fuga fuga.${tag_parts[1...2]}.__TAG_PARTS[1...2]__")
|
78
|
+
"not regexp 1" => "rewriterule1 hoge hoge.${tag_parts[0..2]}.__TAG_PARTS[0..2]__",
|
79
|
+
"not regexp 2" => "rewriterule1 fuga fuga.${tag_parts[1...2]}.__TAG_PARTS[1...2]__")
|
81
80
|
test "invalid" do |conf|
|
82
|
-
assert_raise(Fluent::ConfigError)
|
81
|
+
assert_raise(Fluent::ConfigError) do
|
83
82
|
create_driver(conf)
|
84
|
-
|
83
|
+
end
|
85
84
|
end
|
86
85
|
|
87
86
|
test "valid" do
|
@@ -94,348 +93,136 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
94
93
|
end
|
95
94
|
end
|
96
95
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
d1.
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
end
|
117
|
-
|
118
|
-
test "indent and capitalize option" do
|
119
|
-
d1 = create_driver(CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION, 'input.access')
|
120
|
-
d1.run do
|
121
|
-
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
122
|
-
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
123
|
-
d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
124
|
-
d1.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
125
|
-
end
|
126
|
-
emits = d1.emits
|
127
|
-
assert_equal 4, emits.length
|
128
|
-
assert_equal 'site.Google', emits[0][0] # tag
|
129
|
-
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
130
|
-
assert_equal 'news.google.com', emits[1][2]['domain']
|
131
|
-
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
132
|
-
assert_equal 'agent.Googlebot-Foobar', emits[3][0] #tag
|
133
|
-
end
|
134
|
-
|
135
|
-
test "remove_tag_prefix" do
|
136
|
-
d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX, 'input.access')
|
137
|
-
d1.run do
|
138
|
-
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
139
|
-
end
|
140
|
-
emits = d1.emits
|
141
|
-
assert_equal 1, emits.length
|
142
|
-
assert_equal 'access', emits[0][0] # tag
|
143
|
-
end
|
144
|
-
|
145
|
-
test "remove_tag_prefix with dot" do
|
146
|
-
d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX_WITH_DOT, 'input.access')
|
147
|
-
d1.run do
|
148
|
-
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
149
|
-
end
|
150
|
-
emits = d1.emits
|
151
|
-
assert_equal 1, emits.length
|
152
|
-
assert_equal 'access', emits[0][0] # tag
|
153
|
-
end
|
154
|
-
|
155
|
-
test "short_hostname" do
|
156
|
-
d1 = create_driver(CONFIG_SHORT_HOSTNAME, 'input.access')
|
157
|
-
d1.run do
|
158
|
-
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
159
|
-
end
|
160
|
-
emits = d1.emits
|
161
|
-
assert_equal 1, emits.length
|
162
|
-
assert_equal `hostname -s`.chomp, emits[0][0] # tag
|
163
|
-
end
|
164
|
-
|
165
|
-
test "non_matching" do
|
166
|
-
d1 = create_driver(CONFIG_NON_MATCHING, 'input.access')
|
167
|
-
d1.run do
|
168
|
-
d1.emit({'domain' => 'www.google.com'})
|
169
|
-
d1.emit({'path' => '/'})
|
170
|
-
d1.emit({'domain' => 'maps.google.com'})
|
171
|
-
end
|
172
|
-
emits = d1.emits
|
173
|
-
assert_equal 3, emits.length
|
174
|
-
assert_equal 'start_with_www', emits[0][0] # tag
|
175
|
-
assert_equal 'not_start_with_www', emits[1][0] # tag
|
176
|
-
assert_equal 'not_start_with_www', emits[2][0] # tag
|
177
|
-
end
|
178
|
-
|
179
|
-
test "jump_index" do
|
180
|
-
d1 = create_driver(CONFIG_JUMP_INDEX, 'input.access')
|
181
|
-
d1.run do
|
182
|
-
d1.emit({'domain' => 'www.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
183
|
-
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
184
|
-
end
|
185
|
-
emits = d1.emits
|
186
|
-
assert_equal 2, emits.length
|
187
|
-
assert_equal 'site.Google', emits[0][0] # tag
|
188
|
-
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
189
|
-
end
|
190
|
-
|
191
|
-
test "split_by_tag" do
|
192
|
-
d1 = create_driver(CONFIG_SPLIT_BY_TAG, 'game.production.api')
|
193
|
-
d1.run do
|
194
|
-
d1.emit({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
|
195
|
-
d1.emit({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
|
196
|
-
d1.emit({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
|
197
|
-
d1.emit({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
|
198
|
-
d1.emit({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
|
199
|
-
end
|
200
|
-
emits = d1.emits
|
201
|
-
assert_equal 5, emits.length
|
202
|
-
assert_equal 'application.game.chaos_server', emits[0][0]
|
203
|
-
assert_equal 'application.game.chaos_server', emits[1][0]
|
204
|
-
assert_equal 'application.production.future_server', emits[2][0]
|
205
|
-
assert_equal 'vip.production.remember_love', emits[3][0]
|
206
|
-
assert_equal 'api.game.production', emits[4][0]
|
207
|
-
end
|
208
|
-
|
209
|
-
test "invalid byte (UTF-8)" do
|
210
|
-
invalid_utf8 = "\xff".force_encoding('UTF-8')
|
211
|
-
d1 = create_driver(CONFIG_INVALID_BYTE, 'input.activity')
|
212
|
-
d1.run do
|
213
|
-
d1.emit({'client_name' => invalid_utf8})
|
214
|
-
end
|
215
|
-
emits = d1.emits
|
216
|
-
assert_equal 1, emits.length
|
217
|
-
assert_equal "app.?", emits[0][0]
|
218
|
-
assert_equal invalid_utf8, emits[0][2]['client_name']
|
219
|
-
end
|
96
|
+
def test_emit
|
97
|
+
d1 = create_driver(CONFIG)
|
98
|
+
d1.run(default_tag: "input.access") do
|
99
|
+
d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
100
|
+
d1.feed({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
101
|
+
d1.feed({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
102
|
+
d1.feed({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
103
|
+
d1.feed({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
104
|
+
d1.feed({'domain' => 'noop.example.com'}) # to be ignored
|
105
|
+
end
|
106
|
+
events = d1.events
|
107
|
+
assert_equal 5, events.length
|
108
|
+
assert_equal 'site.Google', events[0][0] # tag
|
109
|
+
assert_equal 'site.GoogleNews', events[1][0] # tag
|
110
|
+
assert_equal 'news.google.com', events[1][2]['domain']
|
111
|
+
assert_equal 'agent.MacOSX', events[2][0] #tag
|
112
|
+
assert_equal 'agent.Googlebot-FooBar', events[3][0] #tag
|
113
|
+
assert_equal 'site.input.access.tagtest', events[4][0] #tag
|
114
|
+
end
|
220
115
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
d1.
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
116
|
+
def test_emit2_indent_and_capitalize_option
|
117
|
+
d1 = create_driver(CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION)
|
118
|
+
d1.run(default_tag: "input.access") do
|
119
|
+
d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
120
|
+
d1.feed({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
121
|
+
d1.feed({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
122
|
+
d1.feed({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
123
|
+
end
|
124
|
+
events = d1.events
|
125
|
+
assert_equal 4, events.length
|
126
|
+
assert_equal 'site.Google', events[0][0] # tag
|
127
|
+
assert_equal 'site.GoogleNews', events[1][0] # tag
|
128
|
+
assert_equal 'news.google.com', events[1][2]['domain']
|
129
|
+
assert_equal 'agent.MacOSX', events[2][0] #tag
|
130
|
+
assert_equal 'agent.Googlebot-Foobar', events[3][0] #tag
|
232
131
|
end
|
233
132
|
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
key domain
|
239
|
-
pattern ^www\.google\.com$
|
240
|
-
tag site.Google
|
241
|
-
</rule>
|
242
|
-
<rule>
|
243
|
-
key domain
|
244
|
-
pattern ^news\.google\.com$
|
245
|
-
tag site.GoogleNews
|
246
|
-
</rule>
|
247
|
-
<rule>
|
248
|
-
key agent
|
249
|
-
pattern .* Mac OS X .*
|
250
|
-
tag agent.MacOSX
|
251
|
-
</rule>
|
252
|
-
<rule>
|
253
|
-
key agent
|
254
|
-
pattern (Googlebot|CustomBot)-([a-zA-Z]+)
|
255
|
-
tag agent.$1-$2
|
256
|
-
</rule>
|
257
|
-
<rule>
|
258
|
-
key domain
|
259
|
-
pattern ^(tagtest)\.google\.com$
|
260
|
-
tag site.${tag}.$1
|
261
|
-
</rule>
|
262
|
-
]
|
263
|
-
d = create_driver(config, 'input.access')
|
264
|
-
d.run do
|
265
|
-
d.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
266
|
-
d.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
267
|
-
d.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
268
|
-
d.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
269
|
-
d.emit({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
270
|
-
d.emit({'domain' => 'noop.example.com'}) # to be ignored
|
271
|
-
end
|
272
|
-
emits = d.emits
|
273
|
-
assert_equal 5, emits.length
|
274
|
-
assert_equal 'site.Google', emits[0][0] # tag
|
275
|
-
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
276
|
-
assert_equal 'news.google.com', emits[1][2]['domain']
|
277
|
-
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
278
|
-
assert_equal 'agent.Googlebot-FooBar', emits[3][0] #tag
|
279
|
-
assert_equal 'site.input.access.tagtest', emits[4][0] #tag
|
280
|
-
end
|
281
|
-
test "remove_tag_prefix" do
|
282
|
-
config = %[
|
283
|
-
remove_tag_prefix input
|
284
|
-
<rule>
|
285
|
-
key domain
|
286
|
-
pattern ^www\.google\.com$
|
287
|
-
tag ${tag}
|
288
|
-
</rule>
|
289
|
-
]
|
290
|
-
d = create_driver(config, 'input.access')
|
291
|
-
d.run do
|
292
|
-
d.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
293
|
-
end
|
294
|
-
emits = d.emits
|
295
|
-
assert_equal 1, emits.length
|
296
|
-
assert_equal 'access', emits[0][0] # tag
|
133
|
+
def test_emit3_remove_tag_prefix
|
134
|
+
d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX)
|
135
|
+
d1.run(default_tag: "input.access") do
|
136
|
+
d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
297
137
|
end
|
138
|
+
events = d1.events
|
139
|
+
assert_equal 1, events.length
|
140
|
+
assert_equal 'access', events[0][0] # tag
|
141
|
+
end
|
298
142
|
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
key domain
|
304
|
-
pattern ^www\.google\.com$
|
305
|
-
tag ${tag}
|
306
|
-
</rule>
|
307
|
-
]
|
308
|
-
d = create_driver(config, 'input.access')
|
309
|
-
d.run do
|
310
|
-
d.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
311
|
-
end
|
312
|
-
emits = d.emits
|
313
|
-
assert_equal 1, emits.length
|
314
|
-
assert_equal 'access', emits[0][0] # tag
|
143
|
+
def test_emit4_remove_tag_prefix_with_dot
|
144
|
+
d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX_WITH_DOT)
|
145
|
+
d1.run(default_tag: "input.access") do
|
146
|
+
d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
315
147
|
end
|
148
|
+
events = d1.events
|
149
|
+
assert_equal 1, events.length
|
150
|
+
assert_equal 'access', events[0][0] # tag
|
151
|
+
end
|
316
152
|
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
<rule>
|
322
|
-
key domain
|
323
|
-
pattern ^www\.google\.com$
|
324
|
-
tag ${hostname}
|
325
|
-
</rule>
|
326
|
-
]
|
327
|
-
d = create_driver(config, 'input.access')
|
328
|
-
d.run do
|
329
|
-
d.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
330
|
-
end
|
331
|
-
emits = d.emits
|
332
|
-
assert_equal 1, emits.length
|
333
|
-
assert_equal `hostname -s`.chomp, emits[0][0] # tag
|
153
|
+
def test_emit5_short_hostname
|
154
|
+
d1 = create_driver(CONFIG_SHORT_HOSTNAME)
|
155
|
+
d1.run(default_tag: "input.access") do
|
156
|
+
d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
334
157
|
end
|
158
|
+
events = d1.events
|
159
|
+
assert_equal 1, events.length
|
160
|
+
assert_equal `hostname -s`.chomp, events[0][0] # tag
|
161
|
+
end
|
335
162
|
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
]
|
350
|
-
d = create_driver(config, 'input.access')
|
351
|
-
d.run do
|
352
|
-
d.emit({'domain' => 'www.google.com'})
|
353
|
-
d.emit({'path' => '/'})
|
354
|
-
d.emit({'domain' => 'maps.google.com'})
|
355
|
-
end
|
356
|
-
emits = d.emits
|
357
|
-
assert_equal 3, emits.length
|
358
|
-
assert_equal 'start_with_www', emits[0][0] # tag
|
359
|
-
assert_equal 'not_start_with_www', emits[1][0] # tag
|
360
|
-
assert_equal 'not_start_with_www', emits[2][0] # tag
|
361
|
-
end
|
163
|
+
def test_emit6_non_matching
|
164
|
+
d1 = create_driver(CONFIG_NON_MATCHING)
|
165
|
+
d1.run(default_tag: "input.access") do
|
166
|
+
d1.feed({'domain' => 'www.google.com'})
|
167
|
+
d1.feed({'path' => '/'})
|
168
|
+
d1.feed({'domain' => 'maps.google.com'})
|
169
|
+
end
|
170
|
+
events = d1.events
|
171
|
+
assert_equal 3, events.length
|
172
|
+
assert_equal 'start_with_www', events[0][0] # tag
|
173
|
+
assert_equal 'not_start_with_www', events[1][0] # tag
|
174
|
+
assert_equal 'not_start_with_www', events[2][0] # tag
|
175
|
+
end
|
362
176
|
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
tag vip.${tag_parts[1]}.remember_love
|
369
|
-
</rule>
|
370
|
-
<rule>
|
371
|
-
key user_name
|
372
|
-
pattern ^Harlock$
|
373
|
-
tag ${tag_parts[2]}.${tag_parts[0]}.${tag_parts[1]}
|
374
|
-
</rule>
|
375
|
-
<rule>
|
376
|
-
key world
|
377
|
-
pattern ^(alice|chaos)$
|
378
|
-
tag application.${tag_parts[0]}.$1_server
|
379
|
-
</rule>
|
380
|
-
<rule>
|
381
|
-
key world
|
382
|
-
pattern ^[a-z]+$
|
383
|
-
tag application.${tag_parts[1]}.future_server
|
384
|
-
</rule>
|
385
|
-
]
|
386
|
-
d = create_driver(config, 'game.production.api')
|
387
|
-
d.run do
|
388
|
-
d.emit({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
|
389
|
-
d.emit({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
|
390
|
-
d.emit({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
|
391
|
-
d.emit({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
|
392
|
-
d.emit({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
|
393
|
-
end
|
394
|
-
emits = d.emits
|
395
|
-
assert_equal 5, emits.length
|
396
|
-
assert_equal 'application.game.chaos_server', emits[0][0]
|
397
|
-
assert_equal 'application.game.chaos_server', emits[1][0]
|
398
|
-
assert_equal 'application.production.future_server', emits[2][0]
|
399
|
-
assert_equal 'vip.production.remember_love', emits[3][0]
|
400
|
-
assert_equal 'api.game.production', emits[4][0]
|
177
|
+
def test_emit7_jump_index
|
178
|
+
d1 = create_driver(CONFIG_JUMP_INDEX)
|
179
|
+
d1.run(default_tag: "input.access") do
|
180
|
+
d1.feed({'domain' => 'www.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
181
|
+
d1.feed({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
401
182
|
end
|
183
|
+
events = d1.events
|
184
|
+
assert_equal 2, events.length
|
185
|
+
assert_equal 'site.Google', events[0][0] # tag
|
186
|
+
assert_equal 'site.GoogleNews', events[1][0] # tag
|
187
|
+
end
|
402
188
|
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
end
|
189
|
+
def test_emit8_split_by_tag
|
190
|
+
d1 = create_driver(CONFIG_SPLIT_BY_TAG)
|
191
|
+
d1.run(default_tag: "game.production.api") do
|
192
|
+
d1.feed({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
|
193
|
+
d1.feed({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
|
194
|
+
d1.feed({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
|
195
|
+
d1.feed({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
|
196
|
+
d1.feed({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
|
197
|
+
end
|
198
|
+
events = d1.events
|
199
|
+
assert_equal 5, events.length
|
200
|
+
assert_equal 'application.game.chaos_server', events[0][0]
|
201
|
+
assert_equal 'application.game.chaos_server', events[1][0]
|
202
|
+
assert_equal 'application.production.future_server', events[2][0]
|
203
|
+
assert_equal 'vip.production.remember_love', events[3][0]
|
204
|
+
assert_equal 'api.game.production', events[4][0]
|
205
|
+
end
|
421
206
|
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
207
|
+
def test_emit9_invalid_byte
|
208
|
+
invalid_utf8 = "\xff".force_encoding('UTF-8')
|
209
|
+
d1 = create_driver(CONFIG_INVALID_BYTE)
|
210
|
+
d1.run(default_tag: "input.activity") do
|
211
|
+
d1.feed({'client_name' => invalid_utf8})
|
212
|
+
end
|
213
|
+
events = d1.events
|
214
|
+
assert_equal 1, events.length
|
215
|
+
assert_equal "app.?", events[0][0]
|
216
|
+
assert_equal invalid_utf8, events[0][2]['client_name']
|
217
|
+
|
218
|
+
invalid_ascii = "\xff".force_encoding('US-ASCII')
|
219
|
+
d1 = create_driver(CONFIG_INVALID_BYTE)
|
220
|
+
d1.run(default_tag: "input.activity") do
|
221
|
+
d1.feed({'client_name' => invalid_ascii})
|
222
|
+
end
|
223
|
+
events = d1.events
|
224
|
+
assert_equal 1, events.length
|
225
|
+
assert_equal "app.?", events[0][0]
|
226
|
+
assert_equal invalid_ascii, events[0][2]['client_name']
|
440
227
|
end
|
441
228
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rewrite-tag-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kentaro Yoshida
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-10-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -42,22 +42,16 @@ dependencies:
|
|
42
42
|
name: fluentd
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: 0.12.0
|
48
|
-
- - "<"
|
45
|
+
- - "~>"
|
49
46
|
- !ruby/object:Gem::Version
|
50
|
-
version: 0.14.
|
47
|
+
version: 0.14.2
|
51
48
|
type: :runtime
|
52
49
|
prerelease: false
|
53
50
|
version_requirements: !ruby/object:Gem::Requirement
|
54
51
|
requirements:
|
55
|
-
- - "
|
52
|
+
- - "~>"
|
56
53
|
- !ruby/object:Gem::Version
|
57
|
-
version: 0.
|
58
|
-
- - "<"
|
59
|
-
- !ruby/object:Gem::Version
|
60
|
-
version: 0.14.0
|
54
|
+
version: 0.14.2
|
61
55
|
description:
|
62
56
|
email:
|
63
57
|
- y.ken.studio@gmail.com
|
@@ -92,9 +86,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
92
86
|
version: '0'
|
93
87
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
94
88
|
requirements:
|
95
|
-
- - "
|
89
|
+
- - ">"
|
96
90
|
- !ruby/object:Gem::Version
|
97
|
-
version:
|
91
|
+
version: 1.3.1
|
98
92
|
requirements: []
|
99
93
|
rubyforge_project:
|
100
94
|
rubygems_version: 2.6.13
|