fluent-plugin-rewrite-tag-filter 1.5.6 → 1.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +2 -0
- data/README.md +105 -30
- data/Rakefile +2 -1
- data/example.conf +74 -26
- data/example2.conf +41 -13
- data/fluent-plugin-rewrite-tag-filter.gemspec +3 -3
- data/lib/fluent/plugin/out_rewrite_tag_filter.rb +29 -14
- data/test/plugin/test_out_rewrite_tag_filter.rb +351 -141
- metadata +7 -9
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 20d7ee24c01a68bb2680c05c1d92149b26c33c2d
|
|
4
|
+
data.tar.gz: 28196b54ddfc97307dd8caf397bd0e479145708e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 32ecd39a3f3060c68bc717e9ac8945b80234ebd07eca0230b197842b1b02ef32a932eb4ec6cf0d7eda8967b9c45cbb9b06409009f6e77997de95bf96b65dbc97
|
|
7
|
+
data.tar.gz: 2bf4b43246577b27e5777328f5da2987ed1ee9978b2a4e7d3c2d5d89853e1a39fe904718093c0776cd8082eda5530eefa96e777c814f44d0ad233f88005a9036
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
|
@@ -24,24 +24,20 @@ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
|
|
|
24
24
|
|
|
25
25
|
## Configuration
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
* **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
|
|
28
|
+
* Deprecated: Use \<rule\> section
|
|
29
|
+
* **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
|
|
30
|
+
* Default value: no
|
|
31
|
+
* **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
|
|
32
|
+
* **hostname_command** (string) (optional): Override hostname command for placeholder. (see the section of "Tag placeholder")
|
|
33
|
+
* Default value: `hostname`.
|
|
28
34
|
|
|
29
|
-
|
|
30
|
-
rewriterule<num> <attribute> <regex_pattern> <new_tag>
|
|
31
|
-
|
|
32
|
-
# Optional: Capitalize letter for every matched regex backreference. (ex: maps -> Maps)
|
|
33
|
-
# for more details, see usage.
|
|
34
|
-
capitalize_regex_backreference <yes/no> (default no)
|
|
35
|
-
|
|
36
|
-
# Optional: remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
|
|
37
|
-
remove_tag_prefix <string>
|
|
35
|
+
### \<rule\> section (optional) (multiple)
|
|
38
36
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
log_level <string> (default info)
|
|
44
|
-
```
|
|
37
|
+
* **key** (string) (required): The field name to which the regular expression is applied
|
|
38
|
+
* **pattern** (regexp) (required): The regular expression
|
|
39
|
+
* **tag** (string) (required): New tag
|
|
40
|
+
* **invert** (bool) (optional): If true, rewrite tag when unmatched pattern
|
|
45
41
|
|
|
46
42
|
### Usage
|
|
47
43
|
|
|
@@ -58,19 +54,49 @@ It's a sample to exclude some static file log before split tag by domain.
|
|
|
58
54
|
</source>
|
|
59
55
|
|
|
60
56
|
# "capitalize_regex_backreference yes" affects converting every matched first letter of backreference to upper case. ex: maps -> Maps
|
|
61
|
-
# At
|
|
62
|
-
# At
|
|
63
|
-
# At
|
|
57
|
+
# At 2nd <rule>, redirect to tag named "clear" which unmatched for status code 200.
|
|
58
|
+
# At 3rd <rule>, redirect to tag named "clear" which is not end with ".com"
|
|
59
|
+
# At 6th <rule>, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
|
|
64
60
|
<match td.apache.access>
|
|
65
61
|
@type rewrite_tag_filter
|
|
66
62
|
capitalize_regex_backreference yes
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
63
|
+
<rule>
|
|
64
|
+
key path
|
|
65
|
+
pattern \.(gif|jpe?g|png|pdf|zip)$
|
|
66
|
+
tag clear
|
|
67
|
+
</rule>
|
|
68
|
+
<rule>
|
|
69
|
+
key status
|
|
70
|
+
pattern ^200$
|
|
71
|
+
tag clear
|
|
72
|
+
invert true
|
|
73
|
+
</rule>
|
|
74
|
+
<rule>
|
|
75
|
+
key domain
|
|
76
|
+
pattern ^.+\.com$
|
|
77
|
+
tag clear
|
|
78
|
+
invert true
|
|
79
|
+
</rule>
|
|
80
|
+
<rule>
|
|
81
|
+
key domain
|
|
82
|
+
pattern ^maps\.example\.com$
|
|
83
|
+
tag site.ExampleMaps
|
|
84
|
+
</rule>
|
|
85
|
+
<rule>
|
|
86
|
+
key domain
|
|
87
|
+
pattern ^news\.example\.com$
|
|
88
|
+
tag site.ExampleNews
|
|
89
|
+
</rule>
|
|
90
|
+
<rule>
|
|
91
|
+
key domain
|
|
92
|
+
pattern ^(mail)\.(example)\.com$
|
|
93
|
+
tag site.$2$1
|
|
94
|
+
</rule>
|
|
95
|
+
<rule>
|
|
96
|
+
key domain
|
|
97
|
+
pattern .+
|
|
98
|
+
tag site.unmatched
|
|
99
|
+
</rule>
|
|
74
100
|
</match>
|
|
75
101
|
|
|
76
102
|
<match site.*>
|
|
@@ -116,6 +142,39 @@ $ tailf /var/log/td-agent/td-agent.log
|
|
|
116
142
|
2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [5, "domain", /.+/, "site.unmatched"]
|
|
117
143
|
```
|
|
118
144
|
|
|
145
|
+
### Nested attributes
|
|
146
|
+
|
|
147
|
+
You can handle nested attributes using [filter_record_transformer](https://docs.fluentd.org/v0.14/articles/filter_record_transformer).
|
|
148
|
+
|
|
149
|
+
```
|
|
150
|
+
<filter kubernetes.**>
|
|
151
|
+
@type kubernetes_metadata
|
|
152
|
+
</filter>
|
|
153
|
+
<filter kubernetes.**>
|
|
154
|
+
@type record_transformer
|
|
155
|
+
enable_ruby
|
|
156
|
+
<record>
|
|
157
|
+
kubernetes_namespace ${record["kubernetes"]["namespace"]}
|
|
158
|
+
</record>
|
|
159
|
+
</filter>
|
|
160
|
+
<match kubernetes.**>
|
|
161
|
+
@type rewrite_tag_filter
|
|
162
|
+
rewriterule1 kubernetes_namespace (.+) $1.${tag}
|
|
163
|
+
</match>
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
{
|
|
168
|
+
"kubernetes": {
|
|
169
|
+
"namespace": "default"
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
When original tag is `kubernetes.var.log`, this will be converted to `default.kubernetes.var.log`
|
|
175
|
+
|
|
176
|
+
See also #13.
|
|
177
|
+
|
|
119
178
|
### Tag placeholder
|
|
120
179
|
|
|
121
180
|
It is supported these placeholder for new_tag (rewrited tag).
|
|
@@ -152,27 +211,43 @@ It's a sample to rewrite a tag with placeholder.
|
|
|
152
211
|
# It will get "rewrited.access.ExampleMail"
|
|
153
212
|
<match apache.access>
|
|
154
213
|
@type rewrite_tag_filter
|
|
155
|
-
rewriterule1 domain ^(mail)\.(example)\.com$ rewrited.${tag}.$2$1
|
|
156
214
|
remove_tag_prefix apache
|
|
215
|
+
<rule>
|
|
216
|
+
key domain
|
|
217
|
+
pattern ^(mail)\.(example)\.com$
|
|
218
|
+
tag rewrited.${tag}.$2$1
|
|
219
|
+
</rule>
|
|
157
220
|
</match>
|
|
158
221
|
|
|
159
222
|
# It will get "rewrited.ExampleMail.app30-124.foo.com" when hostname is "app30-124.foo.com"
|
|
160
223
|
<match apache.access>
|
|
161
224
|
@type rewrite_tag_filter
|
|
162
|
-
|
|
225
|
+
<rule>
|
|
226
|
+
key domain
|
|
227
|
+
pattern ^(mail)\.(example)\.com$
|
|
228
|
+
tag rewrited.$2$1.${hostname}
|
|
229
|
+
</rule>
|
|
163
230
|
</match>
|
|
164
231
|
|
|
165
232
|
# It will get "rewrited.ExampleMail.app30-124" when hostname is "app30-124.foo.com"
|
|
166
233
|
<match apache.access>
|
|
167
234
|
@type rewrite_tag_filter
|
|
168
|
-
rewriterule1 domain ^(mail)\.(example)\.com$ rewrited.$2$1.${hostname}
|
|
169
235
|
hostname_command hostname -s
|
|
236
|
+
<rule>
|
|
237
|
+
key domain
|
|
238
|
+
pattern ^(mail)\.(example)\.com$
|
|
239
|
+
tag rewrited.$2$1.${hostname}
|
|
240
|
+
</rule>
|
|
170
241
|
</match>
|
|
171
242
|
|
|
172
243
|
# It will get "rewrited.game.pool"
|
|
173
244
|
<match app.game.pool.activity>
|
|
174
245
|
@type rewrite_tag_filter
|
|
175
|
-
|
|
246
|
+
<rule>
|
|
247
|
+
key domain
|
|
248
|
+
pattern ^.+$
|
|
249
|
+
tag rewrited.${tag_parts[1]}.${tag_parts[2]}
|
|
250
|
+
</rule>
|
|
176
251
|
</match>
|
|
177
252
|
```
|
|
178
253
|
|
data/Rakefile
CHANGED
data/example.conf
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
# growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
|
|
8
8
|
|
|
9
9
|
<source>
|
|
10
|
-
type tail
|
|
10
|
+
@type tail
|
|
11
11
|
path /var/log/httpd/access_log
|
|
12
12
|
format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
|
|
13
13
|
time_format %d/%b/%Y:%H:%M:%S %z
|
|
@@ -18,16 +18,24 @@
|
|
|
18
18
|
|
|
19
19
|
# Extract specified virtual domain
|
|
20
20
|
<match td.apache.access>
|
|
21
|
-
type copy
|
|
21
|
+
@type copy
|
|
22
22
|
<store>
|
|
23
|
-
type rewrite_tag_filter
|
|
23
|
+
@type rewrite_tag_filter
|
|
24
24
|
capitalize_regex_backreference yes
|
|
25
|
-
|
|
25
|
+
<rule>
|
|
26
|
+
key domain
|
|
27
|
+
pattern ^(maps|news|mail)\.google\.com$
|
|
28
|
+
tag site.Google$1
|
|
29
|
+
</rule>
|
|
26
30
|
</store>
|
|
27
31
|
<store>
|
|
28
|
-
type rewrite_tag_filter
|
|
32
|
+
@type rewrite_tag_filter
|
|
29
33
|
capitalize_regex_backreference yes
|
|
30
|
-
|
|
34
|
+
<rule>
|
|
35
|
+
key domain
|
|
36
|
+
pattern ^(maps)\.google\.com$
|
|
37
|
+
tag sitepath.Google$1
|
|
38
|
+
</rule>
|
|
31
39
|
</store>
|
|
32
40
|
</match>
|
|
33
41
|
|
|
@@ -35,32 +43,72 @@
|
|
|
35
43
|
# Second level analyzing
|
|
36
44
|
<match sitepath.GoogleMaps>
|
|
37
45
|
<store>
|
|
38
|
-
type rewrite_tag_filter
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
46
|
+
@type rewrite_tag_filter
|
|
47
|
+
<rule>
|
|
48
|
+
key path
|
|
49
|
+
pattern ^/labs
|
|
50
|
+
tag site.GoogleMaps.Labs
|
|
51
|
+
</rule>
|
|
52
|
+
<rule>
|
|
53
|
+
key path
|
|
54
|
+
pattern ^/static/\d+
|
|
55
|
+
tag site.GoogleMaps.Static
|
|
56
|
+
</rule>
|
|
57
|
+
<rule>
|
|
58
|
+
key path
|
|
59
|
+
pattern ^/help/maps/(streetview|getmaps)
|
|
60
|
+
tag site.GoogleMaps.Help
|
|
61
|
+
</rule>
|
|
42
62
|
</store>
|
|
43
63
|
<store>
|
|
44
|
-
type rewrite_tag_filter
|
|
45
|
-
|
|
46
|
-
|
|
64
|
+
@type rewrite_tag_filter
|
|
65
|
+
<rule>
|
|
66
|
+
key referer
|
|
67
|
+
pattern headlines\.yahoo\.co\.jp
|
|
68
|
+
tag site.GoogleMaps.referer_YahooHeadlines
|
|
69
|
+
</rule>
|
|
70
|
+
<rule>
|
|
71
|
+
key referer
|
|
72
|
+
pattern news\.livedoor\.com
|
|
73
|
+
tag site.GoogleMaps.referer_LivedoorNews
|
|
74
|
+
</rule>
|
|
47
75
|
</store>
|
|
48
76
|
<store>
|
|
49
|
-
type rewrite_tag_filter
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
77
|
+
@type rewrite_tag_filter
|
|
78
|
+
<rule>
|
|
79
|
+
key agent
|
|
80
|
+
pattern Googlebot/
|
|
81
|
+
tag site.GoogleMaps.agent_Googlebot
|
|
82
|
+
</rule>
|
|
83
|
+
<rule>
|
|
84
|
+
key agent
|
|
85
|
+
pattern ^.* iPhone .+Googlebot-Mobile/.*$
|
|
86
|
+
tag site.GoogleMaps.agent_GooglebotSmartphone
|
|
87
|
+
</rule>
|
|
88
|
+
<rule>
|
|
89
|
+
key agent
|
|
90
|
+
pattern Googlebot-Mobile/
|
|
91
|
+
tag site.GoogleMaps.agent_GooglebotMobile
|
|
92
|
+
</rule>
|
|
93
|
+
<rule>
|
|
94
|
+
key agent
|
|
95
|
+
pattern bingbot
|
|
96
|
+
tag site.GoogleMaps.agent_Bingbot
|
|
97
|
+
</rule>
|
|
98
|
+
<rule>
|
|
99
|
+
key agent
|
|
100
|
+
pattern Baiduspider
|
|
101
|
+
tag site.GoogleMaps.agent_Baiduspider
|
|
102
|
+
</rule>
|
|
55
103
|
</store>
|
|
56
104
|
</match>
|
|
57
105
|
|
|
58
106
|
|
|
59
107
|
# Summarize
|
|
60
108
|
<match site.**>
|
|
61
|
-
type copy
|
|
109
|
+
@type copy
|
|
62
110
|
<store>
|
|
63
|
-
type forest
|
|
111
|
+
@type forest
|
|
64
112
|
subtype datacounter
|
|
65
113
|
<template>
|
|
66
114
|
unit minute
|
|
@@ -78,7 +126,7 @@
|
|
|
78
126
|
</template>
|
|
79
127
|
</store>
|
|
80
128
|
<store>
|
|
81
|
-
type forest
|
|
129
|
+
@type forest
|
|
82
130
|
subtype datacounter
|
|
83
131
|
<template>
|
|
84
132
|
unit minute
|
|
@@ -99,7 +147,7 @@
|
|
|
99
147
|
</template>
|
|
100
148
|
</store>
|
|
101
149
|
<store>
|
|
102
|
-
type forest
|
|
150
|
+
@type forest
|
|
103
151
|
subtype datacounter
|
|
104
152
|
<template>
|
|
105
153
|
unit minute
|
|
@@ -119,7 +167,7 @@
|
|
|
119
167
|
|
|
120
168
|
# Graph
|
|
121
169
|
<match gf.responsetime.**>
|
|
122
|
-
type forest
|
|
170
|
+
@type forest
|
|
123
171
|
subtype growthforecast
|
|
124
172
|
remove_prefix gf.responsetime.site
|
|
125
173
|
<template>
|
|
@@ -131,7 +179,7 @@
|
|
|
131
179
|
</match>
|
|
132
180
|
|
|
133
181
|
<match gf.responsecode.**>
|
|
134
|
-
type forest
|
|
182
|
+
@type forest
|
|
135
183
|
subtype growthforecast
|
|
136
184
|
remove_prefix gf.responsecode.site
|
|
137
185
|
<template>
|
|
@@ -143,7 +191,7 @@
|
|
|
143
191
|
</match>
|
|
144
192
|
|
|
145
193
|
<match gf.useragent.**>
|
|
146
|
-
type forest
|
|
194
|
+
@type forest
|
|
147
195
|
subtype growthforecast
|
|
148
196
|
remove_prefix gf.useragent.site
|
|
149
197
|
<template>
|
data/example2.conf
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
# growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
|
|
8
8
|
|
|
9
9
|
<source>
|
|
10
|
-
type tail
|
|
10
|
+
@type tail
|
|
11
11
|
path /var/log/httpd/access_log
|
|
12
12
|
format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
|
|
13
13
|
time_format %d/%b/%Y:%H:%M:%S %z
|
|
@@ -18,26 +18,54 @@
|
|
|
18
18
|
|
|
19
19
|
# Extract specified virtual domain
|
|
20
20
|
<match td.apache.access>
|
|
21
|
-
type rewrite_tag_filter
|
|
22
|
-
|
|
21
|
+
@type rewrite_tag_filter
|
|
22
|
+
<rule>
|
|
23
|
+
key domain
|
|
24
|
+
pattern ^maps\.google\.com$
|
|
25
|
+
tag filter.GoogleMap
|
|
26
|
+
</rule>
|
|
23
27
|
</match>
|
|
24
28
|
|
|
25
29
|
|
|
26
30
|
# Filtering
|
|
27
31
|
<match filter.GoogleMap>
|
|
28
|
-
type rewrite_tag_filter
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
@type rewrite_tag_filter
|
|
33
|
+
<rule>
|
|
34
|
+
key path
|
|
35
|
+
pattern ^/(img|css|js|static|assets)/
|
|
36
|
+
tag clear
|
|
37
|
+
</rule>
|
|
38
|
+
<rule>
|
|
39
|
+
key status
|
|
40
|
+
pattern ^(?!200)$
|
|
41
|
+
tag clear
|
|
42
|
+
</rule>
|
|
43
|
+
<rule>
|
|
44
|
+
key method
|
|
45
|
+
pattern ^(?!GET)$
|
|
46
|
+
tag clear
|
|
47
|
+
</rule>
|
|
48
|
+
<rule>
|
|
49
|
+
key agent
|
|
50
|
+
pattern (spider|bot|crawler|\+http\:)
|
|
51
|
+
tag clear
|
|
52
|
+
</rule>
|
|
53
|
+
<rule>
|
|
54
|
+
key path
|
|
55
|
+
pattern ^/(admin|api|backend)
|
|
56
|
+
tag site.GoogleMap.backend
|
|
57
|
+
</rule>
|
|
58
|
+
<rule>
|
|
59
|
+
key path
|
|
60
|
+
pattern .+
|
|
61
|
+
tag site.GoogleMap.front
|
|
62
|
+
</rule>
|
|
35
63
|
</match>
|
|
36
64
|
|
|
37
65
|
|
|
38
66
|
# Summarize
|
|
39
67
|
<match site.**>
|
|
40
|
-
type forest
|
|
68
|
+
@type forest
|
|
41
69
|
subtype datacounter
|
|
42
70
|
<template>
|
|
43
71
|
unit minute
|
|
@@ -58,7 +86,7 @@
|
|
|
58
86
|
|
|
59
87
|
# Graph
|
|
60
88
|
<match gf.responsetime.**>
|
|
61
|
-
type forest
|
|
89
|
+
@type forest
|
|
62
90
|
subtype growthforecast
|
|
63
91
|
remove_prefix gf.responsetime.site
|
|
64
92
|
<template>
|
|
@@ -72,6 +100,6 @@
|
|
|
72
100
|
|
|
73
101
|
# Clear tag
|
|
74
102
|
<match clear>
|
|
75
|
-
type null
|
|
103
|
+
@type null
|
|
76
104
|
</match>
|
|
77
105
|
|
|
@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |s|
|
|
5
5
|
s.name = "fluent-plugin-rewrite-tag-filter"
|
|
6
|
-
s.version = "1.
|
|
6
|
+
s.version = "1.6.0"
|
|
7
7
|
s.license = "Apache-2.0"
|
|
8
8
|
s.authors = ["Kentaro Yoshida"]
|
|
9
9
|
s.email = ["y.ken.studio@gmail.com"]
|
|
10
|
-
s.homepage = "https://github.com/
|
|
10
|
+
s.homepage = "https://github.com/fluent/fluent-plugin-rewrite-tag-filter"
|
|
11
11
|
s.summary = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Re-emmit a record with rewrited tag when a value matches/unmatches with the regular expression. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, request-uri, regex-backreference and so on with regular expression.}
|
|
12
12
|
|
|
13
13
|
s.files = `git ls-files`.split("\n")
|
|
@@ -17,5 +17,5 @@ Gem::Specification.new do |s|
|
|
|
17
17
|
|
|
18
18
|
s.add_development_dependency "test-unit", ">= 3.1.0"
|
|
19
19
|
s.add_development_dependency "rake"
|
|
20
|
-
s.add_runtime_dependency "fluentd", [">= 0.
|
|
20
|
+
s.add_runtime_dependency "fluentd", [">= 0.12.0", "< 0.14.0"]
|
|
21
21
|
end
|
|
@@ -1,11 +1,6 @@
|
|
|
1
1
|
class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
2
2
|
Fluent::Plugin.register_output('rewrite_tag_filter', self)
|
|
3
3
|
|
|
4
|
-
# To support Fluentd v0.10.57 or earlier
|
|
5
|
-
unless method_defined?(:router)
|
|
6
|
-
define_method("router") { Fluent::Engine }
|
|
7
|
-
end
|
|
8
|
-
|
|
9
4
|
# For fluentd v0.12.16 or earlier
|
|
10
5
|
class << self
|
|
11
6
|
unless method_defined?(:desc)
|
|
@@ -21,13 +16,21 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
|
21
16
|
desc 'Override hostname command for placeholder.'
|
|
22
17
|
config_param :hostname_command, :string, :default => 'hostname'
|
|
23
18
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
19
|
+
config_section :rule, :param_name => :rules, :multi => true do
|
|
20
|
+
desc "The field name to which the regular expression is applied"
|
|
21
|
+
config_param :key, :string
|
|
22
|
+
desc "The regular expression"
|
|
23
|
+
config_param :pattern do |value|
|
|
24
|
+
Regexp.compile(value)
|
|
25
|
+
end
|
|
26
|
+
desc "New tag"
|
|
27
|
+
config_param :tag, :string
|
|
28
|
+
desc "If true, rewrite tag when unmatched pattern"
|
|
29
|
+
config_param :invert, :bool, :default => false
|
|
29
30
|
end
|
|
30
31
|
|
|
32
|
+
MATCH_OPERATOR_EXCLUDE = '!'
|
|
33
|
+
|
|
31
34
|
def initialize
|
|
32
35
|
super
|
|
33
36
|
require 'string/scrub' if RUBY_VERSION.to_f < 2.1
|
|
@@ -40,6 +43,20 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
|
40
43
|
rewriterule_names = []
|
|
41
44
|
@hostname = `#{@hostname_command}`.chomp
|
|
42
45
|
|
|
46
|
+
@rules.each do |rule|
|
|
47
|
+
unless rule.tag.match(/\$\{tag_parts\[\d\.\.\.?\d\]\}/).nil? or rule.tag.match(/__TAG_PARTS\[\d\.\.\.?\d\]__/).nil?
|
|
48
|
+
raise Fluent::ConfigError, "${tag_parts[n]} and __TAG_PARTS[n]__ placeholder does not support range specify at #{rule}"
|
|
49
|
+
end
|
|
50
|
+
invert = rule.invert ? MATCH_OPERATOR_EXCLUDE : ""
|
|
51
|
+
@rewriterules.push([rule.key, rule.pattern, invert, rule.tag])
|
|
52
|
+
rewriterule_names.push(rule.key + invert + rule.pattern.to_s)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
deprecated_rewriterule = conf.keys.detect {|k| k =~ /^rewriterule(\d+)$/ }
|
|
56
|
+
if deprecated_rewriterule
|
|
57
|
+
k = deprecated_rewriterule.split(" ").first
|
|
58
|
+
log.warn "rewrite_tag_filter: [DEPRECATED] Use <rule> section instead of #{k}"
|
|
59
|
+
end
|
|
43
60
|
conf.keys.select{|k| k =~ /^rewriterule(\d+)$/}.sort_by{|i| i.sub('rewriterule', '').to_i}.each do |key|
|
|
44
61
|
rewritekey,regexp,rewritetag = parse_rewriterule(conf[key])
|
|
45
62
|
if regexp.nil? || rewritetag.nil?
|
|
@@ -102,11 +119,9 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
|
|
|
102
119
|
end
|
|
103
120
|
|
|
104
121
|
def regexp_last_match(regexp, rewritevalue)
|
|
105
|
-
|
|
106
|
-
return if regexp.nil?
|
|
122
|
+
if rewritevalue.valid_encoding?
|
|
107
123
|
regexp.match(rewritevalue)
|
|
108
|
-
|
|
109
|
-
raise e unless e.message.index('invalid byte sequence in') == 0
|
|
124
|
+
else
|
|
110
125
|
regexp.match(rewritevalue.scrub('?'))
|
|
111
126
|
end
|
|
112
127
|
end
|
|
@@ -68,164 +68,374 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
|
68
68
|
rewriterule1 client_name (.+) app.$1
|
|
69
69
|
]
|
|
70
70
|
|
|
71
|
-
def create_driver(conf
|
|
71
|
+
def create_driver(conf, tag = 'test')
|
|
72
72
|
Fluent::Test::OutputTestDriver.new(Fluent::RewriteTagFilterOutput, tag).configure(conf)
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
d = create_driver('rewriterule1 hoge hoge.${tag_parts[0..2]}.__TAG_PARTS[0..2]__')
|
|
87
|
-
}
|
|
88
|
-
assert_raise(Fluent::ConfigError) {
|
|
89
|
-
d = create_driver('rewriterule1 fuga fuga.${tag_parts[1...2]}.__TAG_PARTS[1...2]__')
|
|
90
|
-
}
|
|
91
|
-
d = create_driver %[
|
|
92
|
-
rewriterule1 domain ^www.google.com$ site.Google
|
|
93
|
-
rewriterule2 domain ^news.google.com$ site.GoogleNews
|
|
94
|
-
]
|
|
95
|
-
assert_equal 'domain ^www.google.com$ site.Google', d.instance.config['rewriterule1']
|
|
96
|
-
assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.config['rewriterule2']
|
|
97
|
-
end
|
|
75
|
+
sub_test_case "configure" do
|
|
76
|
+
data("empty" => "",
|
|
77
|
+
"missing regexp" => "rewriterule1 foo",
|
|
78
|
+
"missing new tag" => "rewriterule1 foo foo",
|
|
79
|
+
"not regext 1" => "rewriterule1 hoge hoge.${tag_parts[0..2]}.__TAG_PARTS[0..2]__",
|
|
80
|
+
"not regext 2" => "rewriterule1 fuga fuga.${tag_parts[1...2]}.__TAG_PARTS[1...2]__")
|
|
81
|
+
test "invalid" do |conf|
|
|
82
|
+
assert_raise(Fluent::ConfigError) {
|
|
83
|
+
create_driver(conf)
|
|
84
|
+
}
|
|
85
|
+
end
|
|
98
86
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
d1.emit({'domain' => 'noop.example.com'}) # to be ignored
|
|
108
|
-
end
|
|
109
|
-
emits = d1.emits
|
|
110
|
-
assert_equal 5, emits.length
|
|
111
|
-
assert_equal 'site.Google', emits[0][0] # tag
|
|
112
|
-
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
|
113
|
-
assert_equal 'news.google.com', emits[1][2]['domain']
|
|
114
|
-
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
|
115
|
-
assert_equal 'agent.Googlebot-FooBar', emits[3][0] #tag
|
|
116
|
-
assert_equal 'site.input.access.tagtest', emits[4][0] #tag
|
|
87
|
+
test "valid" do
|
|
88
|
+
d = create_driver %[
|
|
89
|
+
rewriterule1 domain ^www.google.com$ site.Google
|
|
90
|
+
rewriterule2 domain ^news.google.com$ site.GoogleNews
|
|
91
|
+
]
|
|
92
|
+
assert_equal 'domain ^www.google.com$ site.Google', d.instance.config['rewriterule1']
|
|
93
|
+
assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.config['rewriterule2']
|
|
94
|
+
end
|
|
117
95
|
end
|
|
118
96
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
d1.
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
97
|
+
sub_test_case "line style config" do
|
|
98
|
+
test "emit simple" do
|
|
99
|
+
d1 = create_driver(CONFIG, 'input.access')
|
|
100
|
+
d1.run do
|
|
101
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
102
|
+
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
|
103
|
+
d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
|
104
|
+
d1.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
|
105
|
+
d1.emit({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
|
106
|
+
d1.emit({'domain' => 'noop.example.com'}) # to be ignored
|
|
107
|
+
end
|
|
108
|
+
emits = d1.emits
|
|
109
|
+
assert_equal 5, emits.length
|
|
110
|
+
assert_equal 'site.Google', emits[0][0] # tag
|
|
111
|
+
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
|
112
|
+
assert_equal 'news.google.com', emits[1][2]['domain']
|
|
113
|
+
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
|
114
|
+
assert_equal 'agent.Googlebot-FooBar', emits[3][0] #tag
|
|
115
|
+
assert_equal 'site.input.access.tagtest', emits[4][0] #tag
|
|
116
|
+
end
|
|
135
117
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
118
|
+
test "indent and capitalize option" do
|
|
119
|
+
d1 = create_driver(CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION, 'input.access')
|
|
120
|
+
d1.run do
|
|
121
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
122
|
+
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
|
123
|
+
d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
|
124
|
+
d1.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
|
125
|
+
end
|
|
126
|
+
emits = d1.emits
|
|
127
|
+
assert_equal 4, emits.length
|
|
128
|
+
assert_equal 'site.Google', emits[0][0] # tag
|
|
129
|
+
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
|
130
|
+
assert_equal 'news.google.com', emits[1][2]['domain']
|
|
131
|
+
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
|
132
|
+
assert_equal 'agent.Googlebot-Foobar', emits[3][0] #tag
|
|
140
133
|
end
|
|
141
|
-
emits = d1.emits
|
|
142
|
-
assert_equal 1, emits.length
|
|
143
|
-
assert_equal 'access', emits[0][0] # tag
|
|
144
|
-
end
|
|
145
134
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
135
|
+
test "remove_tag_prefix" do
|
|
136
|
+
d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX, 'input.access')
|
|
137
|
+
d1.run do
|
|
138
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
139
|
+
end
|
|
140
|
+
emits = d1.emits
|
|
141
|
+
assert_equal 1, emits.length
|
|
142
|
+
assert_equal 'access', emits[0][0] # tag
|
|
150
143
|
end
|
|
151
|
-
emits = d1.emits
|
|
152
|
-
assert_equal 1, emits.length
|
|
153
|
-
assert_equal 'access', emits[0][0] # tag
|
|
154
|
-
end
|
|
155
144
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
145
|
+
test "remove_tag_prefix with dot" do
|
|
146
|
+
d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX_WITH_DOT, 'input.access')
|
|
147
|
+
d1.run do
|
|
148
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
149
|
+
end
|
|
150
|
+
emits = d1.emits
|
|
151
|
+
assert_equal 1, emits.length
|
|
152
|
+
assert_equal 'access', emits[0][0] # tag
|
|
160
153
|
end
|
|
161
|
-
emits = d1.emits
|
|
162
|
-
assert_equal 1, emits.length
|
|
163
|
-
assert_equal `hostname -s`.chomp, emits[0][0] # tag
|
|
164
|
-
end
|
|
165
154
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
assert_equal 'start_with_www', emits[0][0] # tag
|
|
176
|
-
assert_equal 'not_start_with_www', emits[1][0] # tag
|
|
177
|
-
assert_equal 'not_start_with_www', emits[2][0] # tag
|
|
178
|
-
end
|
|
155
|
+
test "short_hostname" do
|
|
156
|
+
d1 = create_driver(CONFIG_SHORT_HOSTNAME, 'input.access')
|
|
157
|
+
d1.run do
|
|
158
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
159
|
+
end
|
|
160
|
+
emits = d1.emits
|
|
161
|
+
assert_equal 1, emits.length
|
|
162
|
+
assert_equal `hostname -s`.chomp, emits[0][0] # tag
|
|
163
|
+
end
|
|
179
164
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
165
|
+
test "non_matching" do
|
|
166
|
+
d1 = create_driver(CONFIG_NON_MATCHING, 'input.access')
|
|
167
|
+
d1.run do
|
|
168
|
+
d1.emit({'domain' => 'www.google.com'})
|
|
169
|
+
d1.emit({'path' => '/'})
|
|
170
|
+
d1.emit({'domain' => 'maps.google.com'})
|
|
171
|
+
end
|
|
172
|
+
emits = d1.emits
|
|
173
|
+
assert_equal 3, emits.length
|
|
174
|
+
assert_equal 'start_with_www', emits[0][0] # tag
|
|
175
|
+
assert_equal 'not_start_with_www', emits[1][0] # tag
|
|
176
|
+
assert_equal 'not_start_with_www', emits[2][0] # tag
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
test "jump_index" do
|
|
180
|
+
d1 = create_driver(CONFIG_JUMP_INDEX, 'input.access')
|
|
181
|
+
d1.run do
|
|
182
|
+
d1.emit({'domain' => 'www.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
183
|
+
d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
|
184
|
+
end
|
|
185
|
+
emits = d1.emits
|
|
186
|
+
assert_equal 2, emits.length
|
|
187
|
+
assert_equal 'site.Google', emits[0][0] # tag
|
|
188
|
+
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
test "split_by_tag" do
|
|
192
|
+
d1 = create_driver(CONFIG_SPLIT_BY_TAG, 'game.production.api')
|
|
193
|
+
d1.run do
|
|
194
|
+
d1.emit({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
|
|
195
|
+
d1.emit({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
|
|
196
|
+
d1.emit({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
|
|
197
|
+
d1.emit({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
|
|
198
|
+
d1.emit({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
|
|
199
|
+
end
|
|
200
|
+
emits = d1.emits
|
|
201
|
+
assert_equal 5, emits.length
|
|
202
|
+
assert_equal 'application.game.chaos_server', emits[0][0]
|
|
203
|
+
assert_equal 'application.game.chaos_server', emits[1][0]
|
|
204
|
+
assert_equal 'application.production.future_server', emits[2][0]
|
|
205
|
+
assert_equal 'vip.production.remember_love', emits[3][0]
|
|
206
|
+
assert_equal 'api.game.production', emits[4][0]
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
test "invalid byte (UTF-8)" do
|
|
210
|
+
invalid_utf8 = "\xff".force_encoding('UTF-8')
|
|
211
|
+
d1 = create_driver(CONFIG_INVALID_BYTE, 'input.activity')
|
|
212
|
+
d1.run do
|
|
213
|
+
d1.emit({'client_name' => invalid_utf8})
|
|
214
|
+
end
|
|
215
|
+
emits = d1.emits
|
|
216
|
+
assert_equal 1, emits.length
|
|
217
|
+
assert_equal "app.?", emits[0][0]
|
|
218
|
+
assert_equal invalid_utf8, emits[0][2]['client_name']
|
|
185
219
|
end
|
|
186
|
-
emits = d1.emits
|
|
187
|
-
assert_equal 2, emits.length
|
|
188
|
-
assert_equal 'site.Google', emits[0][0] # tag
|
|
189
|
-
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
|
190
|
-
end
|
|
191
220
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
d1.
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
d1.
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
assert_equal 'application.game.chaos_server', emits[0][0]
|
|
204
|
-
assert_equal 'application.game.chaos_server', emits[1][0]
|
|
205
|
-
assert_equal 'application.production.future_server', emits[2][0]
|
|
206
|
-
assert_equal 'vip.production.remember_love', emits[3][0]
|
|
207
|
-
assert_equal 'api.game.production', emits[4][0]
|
|
221
|
+
test "invalid byte (US-ASCII)" do
|
|
222
|
+
invalid_ascii = "\xff".force_encoding('US-ASCII')
|
|
223
|
+
d1 = create_driver(CONFIG_INVALID_BYTE, 'input.activity')
|
|
224
|
+
d1.run do
|
|
225
|
+
d1.emit({'client_name' => invalid_ascii})
|
|
226
|
+
end
|
|
227
|
+
emits = d1.emits
|
|
228
|
+
assert_equal 1, emits.length
|
|
229
|
+
assert_equal "app.?", emits[0][0]
|
|
230
|
+
assert_equal invalid_ascii, emits[0][2]['client_name']
|
|
231
|
+
end
|
|
208
232
|
end
|
|
209
233
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
234
|
+
sub_test_case "section style" do
|
|
235
|
+
test "simple" do
|
|
236
|
+
config = %[
|
|
237
|
+
<rule>
|
|
238
|
+
key domain
|
|
239
|
+
pattern ^www\.google\.com$
|
|
240
|
+
tag site.Google
|
|
241
|
+
</rule>
|
|
242
|
+
<rule>
|
|
243
|
+
key domain
|
|
244
|
+
pattern ^news\.google\.com$
|
|
245
|
+
tag site.GoogleNews
|
|
246
|
+
</rule>
|
|
247
|
+
<rule>
|
|
248
|
+
key agent
|
|
249
|
+
pattern .* Mac OS X .*
|
|
250
|
+
tag agent.MacOSX
|
|
251
|
+
</rule>
|
|
252
|
+
<rule>
|
|
253
|
+
key agent
|
|
254
|
+
pattern (Googlebot|CustomBot)-([a-zA-Z]+)
|
|
255
|
+
tag agent.$1-$2
|
|
256
|
+
</rule>
|
|
257
|
+
<rule>
|
|
258
|
+
key domain
|
|
259
|
+
pattern ^(tagtest)\.google\.com$
|
|
260
|
+
tag site.${tag}.$1
|
|
261
|
+
</rule>
|
|
262
|
+
]
|
|
263
|
+
d = create_driver(config, 'input.access')
|
|
264
|
+
d.run do
|
|
265
|
+
d.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
266
|
+
d.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
|
267
|
+
d.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
|
268
|
+
d.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
|
269
|
+
d.emit({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
|
270
|
+
d.emit({'domain' => 'noop.example.com'}) # to be ignored
|
|
271
|
+
end
|
|
272
|
+
emits = d.emits
|
|
273
|
+
assert_equal 5, emits.length
|
|
274
|
+
assert_equal 'site.Google', emits[0][0] # tag
|
|
275
|
+
assert_equal 'site.GoogleNews', emits[1][0] # tag
|
|
276
|
+
assert_equal 'news.google.com', emits[1][2]['domain']
|
|
277
|
+
assert_equal 'agent.MacOSX', emits[2][0] #tag
|
|
278
|
+
assert_equal 'agent.Googlebot-FooBar', emits[3][0] #tag
|
|
279
|
+
assert_equal 'site.input.access.tagtest', emits[4][0] #tag
|
|
280
|
+
end
|
|
281
|
+
test "remove_tag_prefix" do
|
|
282
|
+
config = %[
|
|
283
|
+
remove_tag_prefix input
|
|
284
|
+
<rule>
|
|
285
|
+
key domain
|
|
286
|
+
pattern ^www\.google\.com$
|
|
287
|
+
tag ${tag}
|
|
288
|
+
</rule>
|
|
289
|
+
]
|
|
290
|
+
d = create_driver(config, 'input.access')
|
|
291
|
+
d.run do
|
|
292
|
+
d.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
293
|
+
end
|
|
294
|
+
emits = d.emits
|
|
295
|
+
assert_equal 1, emits.length
|
|
296
|
+
assert_equal 'access', emits[0][0] # tag
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
test "remove_tag_prefix with dot" do
|
|
300
|
+
config = %[
|
|
301
|
+
remove_tag_prefix input.
|
|
302
|
+
<rule>
|
|
303
|
+
key domain
|
|
304
|
+
pattern ^www\.google\.com$
|
|
305
|
+
tag ${tag}
|
|
306
|
+
</rule>
|
|
307
|
+
]
|
|
308
|
+
d = create_driver(config, 'input.access')
|
|
309
|
+
d.run do
|
|
310
|
+
d.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
311
|
+
end
|
|
312
|
+
emits = d.emits
|
|
313
|
+
assert_equal 1, emits.length
|
|
314
|
+
assert_equal 'access', emits[0][0] # tag
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
test "short hostname" do
|
|
318
|
+
config = %[
|
|
319
|
+
remove_tag_prefix input
|
|
320
|
+
hostname_command hostname -s
|
|
321
|
+
<rule>
|
|
322
|
+
key domain
|
|
323
|
+
pattern ^www\.google\.com$
|
|
324
|
+
tag ${hostname}
|
|
325
|
+
</rule>
|
|
326
|
+
]
|
|
327
|
+
d = create_driver(config, 'input.access')
|
|
328
|
+
d.run do
|
|
329
|
+
d.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
|
330
|
+
end
|
|
331
|
+
emits = d.emits
|
|
332
|
+
assert_equal 1, emits.length
|
|
333
|
+
assert_equal `hostname -s`.chomp, emits[0][0] # tag
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
test "non matching" do
|
|
337
|
+
config = %[
|
|
338
|
+
<rule>
|
|
339
|
+
key domain
|
|
340
|
+
pattern ^www\..+$
|
|
341
|
+
tag not_start_with_www
|
|
342
|
+
invert true
|
|
343
|
+
</rule>
|
|
344
|
+
<rule>
|
|
345
|
+
key domain
|
|
346
|
+
pattern ^www\..+$
|
|
347
|
+
tag start_with_www
|
|
348
|
+
</rule>
|
|
349
|
+
]
|
|
350
|
+
d = create_driver(config, 'input.access')
|
|
351
|
+
d.run do
|
|
352
|
+
d.emit({'domain' => 'www.google.com'})
|
|
353
|
+
d.emit({'path' => '/'})
|
|
354
|
+
d.emit({'domain' => 'maps.google.com'})
|
|
355
|
+
end
|
|
356
|
+
emits = d.emits
|
|
357
|
+
assert_equal 3, emits.length
|
|
358
|
+
assert_equal 'start_with_www', emits[0][0] # tag
|
|
359
|
+
assert_equal 'not_start_with_www', emits[1][0] # tag
|
|
360
|
+
assert_equal 'not_start_with_www', emits[2][0] # tag
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
test "split by tag" do
|
|
364
|
+
config = %[
|
|
365
|
+
<rule>
|
|
366
|
+
key user_name
|
|
367
|
+
pattern ^Lynn Minmay$
|
|
368
|
+
tag vip.${tag_parts[1]}.remember_love
|
|
369
|
+
</rule>
|
|
370
|
+
<rule>
|
|
371
|
+
key user_name
|
|
372
|
+
pattern ^Harlock$
|
|
373
|
+
tag ${tag_parts[2]}.${tag_parts[0]}.${tag_parts[1]}
|
|
374
|
+
</rule>
|
|
375
|
+
<rule>
|
|
376
|
+
key world
|
|
377
|
+
pattern ^(alice|chaos)$
|
|
378
|
+
tag application.${tag_parts[0]}.$1_server
|
|
379
|
+
</rule>
|
|
380
|
+
<rule>
|
|
381
|
+
key world
|
|
382
|
+
pattern ^[a-z]+$
|
|
383
|
+
tag application.${tag_parts[1]}.future_server
|
|
384
|
+
</rule>
|
|
385
|
+
]
|
|
386
|
+
d = create_driver(config, 'game.production.api')
|
|
387
|
+
d.run do
|
|
388
|
+
d.emit({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
|
|
389
|
+
d.emit({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
|
|
390
|
+
d.emit({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
|
|
391
|
+
d.emit({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
|
|
392
|
+
d.emit({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
|
|
393
|
+
end
|
|
394
|
+
emits = d.emits
|
|
395
|
+
assert_equal 5, emits.length
|
|
396
|
+
assert_equal 'application.game.chaos_server', emits[0][0]
|
|
397
|
+
assert_equal 'application.game.chaos_server', emits[1][0]
|
|
398
|
+
assert_equal 'application.production.future_server', emits[2][0]
|
|
399
|
+
assert_equal 'vip.production.remember_love', emits[3][0]
|
|
400
|
+
assert_equal 'api.game.production', emits[4][0]
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
test "invalid_byte (UTF-8)" do
|
|
404
|
+
config = %[
|
|
405
|
+
<rule>
|
|
406
|
+
key client_name
|
|
407
|
+
pattern (.+)
|
|
408
|
+
tag app.$1
|
|
409
|
+
</rule>
|
|
410
|
+
]
|
|
411
|
+
invalid_utf8 = "\xff".force_encoding('UTF-8')
|
|
412
|
+
d = create_driver(config, 'input.activity')
|
|
413
|
+
d.run do
|
|
414
|
+
d.emit({'client_name' => invalid_utf8})
|
|
415
|
+
end
|
|
416
|
+
emits = d.emits
|
|
417
|
+
assert_equal 1, emits.length
|
|
418
|
+
assert_equal "app.?", emits[0][0]
|
|
419
|
+
assert_equal invalid_utf8, emits[0][2]['client_name']
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
test "invalid byte (US-ASCII)" do
|
|
423
|
+
config = %[
|
|
424
|
+
<rule>
|
|
425
|
+
key client_name
|
|
426
|
+
pattern (.+)
|
|
427
|
+
tag app.$1
|
|
428
|
+
</rule>
|
|
429
|
+
]
|
|
430
|
+
invalid_ascii = "\xff".force_encoding('US-ASCII')
|
|
431
|
+
d = create_driver(config, 'input.activity')
|
|
432
|
+
d.run do
|
|
433
|
+
d.emit({'client_name' => invalid_ascii})
|
|
434
|
+
end
|
|
435
|
+
emits = d.emits
|
|
436
|
+
assert_equal 1, emits.length
|
|
437
|
+
assert_equal "app.?", emits[0][0]
|
|
438
|
+
assert_equal invalid_ascii, emits[0][2]['client_name']
|
|
439
|
+
end
|
|
230
440
|
end
|
|
231
441
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-rewrite-tag-filter
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kentaro Yoshida
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-11-06 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: test-unit
|
|
@@ -44,7 +44,7 @@ dependencies:
|
|
|
44
44
|
requirements:
|
|
45
45
|
- - ">="
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: 0.
|
|
47
|
+
version: 0.12.0
|
|
48
48
|
- - "<"
|
|
49
49
|
- !ruby/object:Gem::Version
|
|
50
50
|
version: 0.14.0
|
|
@@ -54,7 +54,7 @@ dependencies:
|
|
|
54
54
|
requirements:
|
|
55
55
|
- - ">="
|
|
56
56
|
- !ruby/object:Gem::Version
|
|
57
|
-
version: 0.
|
|
57
|
+
version: 0.12.0
|
|
58
58
|
- - "<"
|
|
59
59
|
- !ruby/object:Gem::Version
|
|
60
60
|
version: 0.14.0
|
|
@@ -77,7 +77,7 @@ files:
|
|
|
77
77
|
- lib/fluent/plugin/out_rewrite_tag_filter.rb
|
|
78
78
|
- test/helper.rb
|
|
79
79
|
- test/plugin/test_out_rewrite_tag_filter.rb
|
|
80
|
-
homepage: https://github.com/
|
|
80
|
+
homepage: https://github.com/fluent/fluent-plugin-rewrite-tag-filter
|
|
81
81
|
licenses:
|
|
82
82
|
- Apache-2.0
|
|
83
83
|
metadata: {}
|
|
@@ -97,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
97
97
|
version: '0'
|
|
98
98
|
requirements: []
|
|
99
99
|
rubyforge_project:
|
|
100
|
-
rubygems_version: 2.
|
|
100
|
+
rubygems_version: 2.6.13
|
|
101
101
|
signing_key:
|
|
102
102
|
specification_version: 4
|
|
103
103
|
summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.
|
|
@@ -105,6 +105,4 @@ summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_r
|
|
|
105
105
|
expression. Also you can change a tag from apache log by domain, status-code(ex.
|
|
106
106
|
500 error), user-agent, request-uri, regex-backreference and so on with regular
|
|
107
107
|
expression.
|
|
108
|
-
test_files:
|
|
109
|
-
- test/helper.rb
|
|
110
|
-
- test/plugin/test_out_rewrite_tag_filter.rb
|
|
108
|
+
test_files: []
|