fluent-plugin-rewrite-tag-filter 2.0.0.rc1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -0
- data/README.md +105 -32
- data/fluent-plugin-rewrite-tag-filter.gemspec +1 -1
- data/lib/fluent/plugin/out_rewrite_tag_filter.rb +28 -44
- data/test/helper.rb +3 -23
- data/test/plugin/test_out_rewrite_tag_filter.rb +224 -188
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2644562a37644d56fa8712f13d46aa0ff42cb9ec
|
4
|
+
data.tar.gz: 22ac2b253497f823babee508e41d695b9bdc8218
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eaa2ee232b41ba0b6f387adb69621e78a411919d8d074c3ce014d344b4211a1dad7d47b61a06698cbc4fa0791de55a87111bf91383c47cee3f56f71299ad53e1
|
7
|
+
data.tar.gz: ff33f9efd2106fc70e85023a139563764aa8414329066459bd9dd177133ee9ff8b9bff20119a50d8621c20573f3d7cdcfc3cb37c1d67a4f6210190b9523bb878
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -14,8 +14,6 @@ user-agent, request-uri, regex-backreference and so on with regular expression.
|
|
14
14
|
| >= 2.0.0 | >= v0.14.2 | >= 2.1 |
|
15
15
|
| < 2.0.0 | >= v0.12.0 | >= 1.9 |
|
16
16
|
|
17
|
-
NOTE: fluent-plugin-rewrite-tag-filter v2.0.0 is now RC. We will release stable v2.0.0 soon.
|
18
|
-
|
19
17
|
## Installation
|
20
18
|
|
21
19
|
Install with `gem`, `fluent-gem` or `td-agent-gem` command as:
|
@@ -33,24 +31,21 @@ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
|
|
33
31
|
|
34
32
|
## Configuration
|
35
33
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
capitalize_regex_backreference <yes/no> (default no)
|
44
|
-
|
45
|
-
# Optional: remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
|
46
|
-
remove_tag_prefix <string>
|
34
|
+
* **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
|
35
|
+
* Deprecated: Use <rule> section
|
36
|
+
* **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
|
37
|
+
* Default value: no
|
38
|
+
* **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
|
39
|
+
* **hostname_command** (string) (optional): Override hostname command for placeholder. (see the section of "Tag placeholder")
|
40
|
+
* Default value: `hostname`.
|
47
41
|
|
48
|
-
|
49
|
-
hostname_command <string>
|
42
|
+
### \<rule\> section (optional) (multiple)
|
50
43
|
|
51
|
-
|
52
|
-
|
53
|
-
|
44
|
+
* **key** (string) (required): The field name to which the regular expression is applied
|
45
|
+
* **pattern** (regexp) (required): The regular expression
|
46
|
+
* **tag** (string) (required): New tag
|
47
|
+
* **invert** (bool) (optional): If true, rewrite tag when unmatch pattern
|
48
|
+
* Default value: `false`
|
54
49
|
|
55
50
|
### Usage
|
56
51
|
|
@@ -67,19 +62,49 @@ It's a sample to exclude some static file log before split tag by domain.
|
|
67
62
|
</source>
|
68
63
|
|
69
64
|
# "capitalize_regex_backreference yes" affects converting every matched first letter of backreference to upper case. ex: maps -> Maps
|
70
|
-
# At
|
71
|
-
# At
|
72
|
-
# At
|
65
|
+
# At 2nd <rule>, redirect to tag named "clear" which unmatched for status code 200.
|
66
|
+
# At 3rd <rule>, redirect to tag named "clear" which is not end with ".com"
|
67
|
+
# At 6th <rule>, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
|
73
68
|
<match td.apache.access>
|
74
69
|
@type rewrite_tag_filter
|
75
70
|
capitalize_regex_backreference yes
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
71
|
+
<rule>
|
72
|
+
key path
|
73
|
+
pattern \.(gif|jpe?g|png|pdf|zip)$
|
74
|
+
tag clear
|
75
|
+
</rule>
|
76
|
+
<rule>
|
77
|
+
key status
|
78
|
+
pattern ^200$
|
79
|
+
tag clear
|
80
|
+
invert true
|
81
|
+
</rule>
|
82
|
+
<rule>
|
83
|
+
key domain
|
84
|
+
pattern ^.+\.com$
|
85
|
+
tag clear
|
86
|
+
invert true
|
87
|
+
</rule>
|
88
|
+
<rule>
|
89
|
+
key domain
|
90
|
+
pattern ^maps\.example\.com$
|
91
|
+
tag site.ExampleMaps
|
92
|
+
</rule>
|
93
|
+
<rule>
|
94
|
+
key domain
|
95
|
+
pattern ^news\.example\.com$
|
96
|
+
tag site.ExampleNews
|
97
|
+
</rule>
|
98
|
+
<rule>
|
99
|
+
key domain
|
100
|
+
pattern ^(mail)\.(example)\.com$
|
101
|
+
tag site.$2$1
|
102
|
+
</rule>
|
103
|
+
<rule>
|
104
|
+
key domain
|
105
|
+
pattern .+
|
106
|
+
tag site.unmatched
|
107
|
+
</rule>
|
83
108
|
</match>
|
84
109
|
|
85
110
|
<match site.*>
|
@@ -125,6 +150,38 @@ $ tailf /var/log/td-agent/td-agent.log
|
|
125
150
|
2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [5, "domain", /.+/, "site.unmatched"]
|
126
151
|
```
|
127
152
|
|
153
|
+
### Nested attributes
|
154
|
+
|
155
|
+
Dot notation:
|
156
|
+
|
157
|
+
```
|
158
|
+
<match kubernetes.**>
|
159
|
+
@type rewrite_tag_filter
|
160
|
+
rewriterule1 $.kubernetes.namespace_name ^(.+)$ $1.${tag}
|
161
|
+
</match>
|
162
|
+
```
|
163
|
+
|
164
|
+
Bracket notation:
|
165
|
+
|
166
|
+
```
|
167
|
+
<match kubernetes.**>
|
168
|
+
@type rewrite_tag_filter
|
169
|
+
rewriterule1 $['kubernetes']['namespace_name'] ^(.+)$ $1.${tag}
|
170
|
+
</match>
|
171
|
+
```
|
172
|
+
|
173
|
+
These example configurations can process nested attributes like following:
|
174
|
+
|
175
|
+
```
|
176
|
+
{
|
177
|
+
"kubernetes": {
|
178
|
+
"namespace_name": "default"
|
179
|
+
}
|
180
|
+
}
|
181
|
+
```
|
182
|
+
|
183
|
+
When original tag is `kubernetes.var.log`, this will be converted to `default.kubernetes.var.log`.
|
184
|
+
|
128
185
|
### Tag placeholder
|
129
186
|
|
130
187
|
It is supported these placeholder for new_tag (rewrited tag).
|
@@ -161,27 +218,43 @@ It's a sample to rewrite a tag with placeholder.
|
|
161
218
|
# It will get "rewrited.access.ExampleMail"
|
162
219
|
<match apache.access>
|
163
220
|
@type rewrite_tag_filter
|
164
|
-
rewriterule1 domain ^(mail)\.(example)\.com$ rewrited.${tag}.$2$1
|
165
221
|
remove_tag_prefix apache
|
222
|
+
<rule>
|
223
|
+
key domain
|
224
|
+
pattern ^(mail)\.(example)\.com$
|
225
|
+
tag rewrited.${tag}.$2$1
|
226
|
+
</rule>
|
166
227
|
</match>
|
167
228
|
|
168
229
|
# It will get "rewrited.ExampleMail.app30-124.foo.com" when hostname is "app30-124.foo.com"
|
169
230
|
<match apache.access>
|
170
231
|
@type rewrite_tag_filter
|
171
|
-
|
232
|
+
<rule>
|
233
|
+
key domain
|
234
|
+
pattern ^(mail)\.(example)\.com$
|
235
|
+
tag rewrited.$2$1.${hostname}
|
236
|
+
</rule>
|
172
237
|
</match>
|
173
238
|
|
174
239
|
# It will get "rewrited.ExampleMail.app30-124" when hostname is "app30-124.foo.com"
|
175
240
|
<match apache.access>
|
176
241
|
@type rewrite_tag_filter
|
177
|
-
rewriterule1 domain ^(mail)\.(example)\.com$ rewrited.$2$1.${hostname}
|
178
242
|
hostname_command hostname -s
|
243
|
+
<rule>
|
244
|
+
key domain
|
245
|
+
pattern ^(mail)\.(example)\.com$
|
246
|
+
tag rewrited.$2$1.${hostname}
|
247
|
+
</rule>
|
179
248
|
</match>
|
180
249
|
|
181
250
|
# It will get "rewrited.game.pool"
|
182
251
|
<match app.game.pool.activity>
|
183
252
|
@type rewrite_tag_filter
|
184
|
-
|
253
|
+
<rule>
|
254
|
+
key domain
|
255
|
+
pattern ^.+$
|
256
|
+
tag rewrited.${tag_parts[1]}.${tag_parts[2]}
|
257
|
+
</rule>
|
185
258
|
</match>
|
186
259
|
```
|
187
260
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-rewrite-tag-filter"
|
6
|
-
s.version = "2.0.0
|
6
|
+
s.version = "2.0.0"
|
7
7
|
s.license = "Apache-2.0"
|
8
8
|
s.authors = ["Kentaro Yoshida"]
|
9
9
|
s.email = ["y.ken.studio@gmail.com"]
|
@@ -3,7 +3,7 @@ require "fluent/plugin/output"
|
|
3
3
|
class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
4
4
|
Fluent::Plugin.register_output('rewrite_tag_filter', self)
|
5
5
|
|
6
|
-
helpers :event_emitter
|
6
|
+
helpers :event_emitter, :record_accessor
|
7
7
|
|
8
8
|
desc 'Capitalize letter for every matched regex backreference.'
|
9
9
|
config_param :capitalize_regex_backreference, :bool, :default => false
|
@@ -12,13 +12,21 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
12
12
|
desc 'Override hostname command for placeholder.'
|
13
13
|
config_param :hostname_command, :string, :default => 'hostname'
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
15
|
+
config_section :rule, param_name: :rules, multi: true do
|
16
|
+
desc "The field name to which the regular expression is applied"
|
17
|
+
config_param :key, :string
|
18
|
+
desc "The regular expression"
|
19
|
+
config_param :pattern do |value|
|
20
|
+
Regexp.compile(value)
|
21
|
+
end
|
22
|
+
desc "New tag"
|
23
|
+
config_param :tag, :string
|
24
|
+
desc "If true, rewrite tag when unmatch pattern"
|
25
|
+
config_param :invert, :bool, default: false
|
20
26
|
end
|
21
27
|
|
28
|
+
MATCH_OPERATOR_EXCLUDE = '!'
|
29
|
+
|
22
30
|
def configure(conf)
|
23
31
|
super
|
24
32
|
|
@@ -26,19 +34,19 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
26
34
|
rewriterule_names = []
|
27
35
|
@hostname = `#{@hostname_command}`.chomp
|
28
36
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
raise Fluent::ConfigError, "failed to parse rewriterules at #{key} #{conf[key]}"
|
37
|
+
@rules.each do |rule|
|
38
|
+
unless rule.tag.match(/\$\{tag_parts\[\d\.\.\.?\d\]\}/).nil? or rule.tag.match(/__TAG_PARTS\[\d\.\.\.?\d\]__/).nil?
|
39
|
+
raise Fluent::ConfigError, "${tag_parts[n]} and __TAG_PARTS[n]__ placeholder does not support range specify at #{rule}"
|
33
40
|
end
|
34
41
|
|
35
|
-
|
36
|
-
|
37
|
-
|
42
|
+
invert = rule.invert ? MATCH_OPERATOR_EXCLUDE : ""
|
43
|
+
@rewriterules.push([record_accessor_create(rule.key), rule.pattern, invert, rule.tag])
|
44
|
+
rewriterule_names.push(rule.key + invert + rule.pattern.to_s)
|
45
|
+
log.info "adding rewrite_tag_filter rule: #{rule.key} #{@rewriterules.last}"
|
46
|
+
end
|
38
47
|
|
39
|
-
|
40
|
-
|
41
|
-
log.info "adding rewrite_tag_filter rule: #{key} #{@rewriterules.last}"
|
48
|
+
if conf.keys.any? {|k| k.start_with?("rewriterule") }
|
49
|
+
raise Fluent::ConfigError, "\"rewriterule<num>\" support has been dropped. Use <rule> section instead."
|
42
50
|
end
|
43
51
|
|
44
52
|
unless @rewriterules.length > 0
|
@@ -68,8 +76,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
68
76
|
|
69
77
|
def rewrite_tag(tag, record)
|
70
78
|
placeholder = get_placeholder(tag)
|
71
|
-
@rewriterules.each do |
|
72
|
-
rewritevalue = record
|
79
|
+
@rewriterules.each do |record_accessor, regexp, match_operator, rewritetag|
|
80
|
+
rewritevalue = record_accessor.call(record).to_s
|
73
81
|
next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
|
74
82
|
last_match = regexp_last_match(regexp, rewritevalue)
|
75
83
|
case match_operator
|
@@ -90,37 +98,13 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
90
98
|
end
|
91
99
|
|
92
100
|
def regexp_last_match(regexp, rewritevalue)
|
93
|
-
|
94
|
-
return if regexp.nil?
|
101
|
+
if rewritevalue.valid_encoding?
|
95
102
|
regexp.match(rewritevalue)
|
96
|
-
|
97
|
-
raise e unless e.message.index('invalid byte sequence in') == 0
|
103
|
+
else
|
98
104
|
regexp.match(rewritevalue.scrub('?'))
|
99
105
|
end
|
100
106
|
end
|
101
107
|
|
102
|
-
def parse_rewriterule(rule)
|
103
|
-
if m = rule.match(/^([^\s]+)\s+(.+?)\s+([^\s]+)$/)
|
104
|
-
return m.captures
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
def trim_regex_quote(regexp)
|
109
|
-
if regexp.start_with?('"') && regexp.end_with?('"')
|
110
|
-
log.info "rewrite_tag_filter: [DEPRECATED] Use ^....$ pattern for partial word match instead of double-quote-delimiter. #{regexp}"
|
111
|
-
regexp = regexp[1..-2]
|
112
|
-
end
|
113
|
-
if regexp.start_with?(MATCH_OPERATOR_EXCLUDE)
|
114
|
-
regexp = regexp[1, regexp.length]
|
115
|
-
end
|
116
|
-
return regexp
|
117
|
-
end
|
118
|
-
|
119
|
-
def get_match_operator(regexp)
|
120
|
-
return MATCH_OPERATOR_EXCLUDE if regexp.start_with?(MATCH_OPERATOR_EXCLUDE)
|
121
|
-
return ''
|
122
|
-
end
|
123
|
-
|
124
108
|
def get_backreference_table(elements)
|
125
109
|
hash_table = Hash.new
|
126
110
|
elements.each.with_index(1) do |value, index|
|
data/test/helper.rb
CHANGED
@@ -1,29 +1,9 @@
|
|
1
|
-
require '
|
2
|
-
require 'bundler'
|
3
|
-
begin
|
4
|
-
Bundler.setup(:default, :development)
|
5
|
-
rescue Bundler::BundlerError => e
|
6
|
-
$stderr.puts e.message
|
7
|
-
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
-
exit e.status_code
|
9
|
-
end
|
1
|
+
require 'bundler/setup'
|
10
2
|
require 'test/unit'
|
11
3
|
|
12
|
-
$LOAD_PATH.unshift(File.join(
|
13
|
-
$LOAD_PATH.unshift(
|
4
|
+
$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
|
5
|
+
$LOAD_PATH.unshift(__dir__)
|
14
6
|
require 'fluent/test'
|
15
7
|
require 'fluent/test/driver/output'
|
16
|
-
unless ENV.has_key?('VERBOSE')
|
17
|
-
nulllogger = Object.new
|
18
|
-
nulllogger.instance_eval {|obj|
|
19
|
-
def method_missing(method, *args)
|
20
|
-
# pass
|
21
|
-
end
|
22
|
-
}
|
23
|
-
$log = nulllogger
|
24
|
-
end
|
25
8
|
|
26
9
|
require 'fluent/plugin/out_rewrite_tag_filter'
|
27
|
-
|
28
|
-
class Test::Unit::TestCase
|
29
|
-
end
|
@@ -5,224 +5,260 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
5
5
|
Fluent::Test.setup
|
6
6
|
end
|
7
7
|
|
8
|
-
|
9
|
-
rewriterule1 domain ^www\.google\.com$ site.Google
|
10
|
-
rewriterule2 domain ^news\.google\.com$ site.GoogleNews
|
11
|
-
rewriterule3 agent .* Mac OS X .* agent.MacOSX
|
12
|
-
rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
|
13
|
-
rewriterule5 domain ^(tagtest)\.google\.com$ site.${tag}.$1
|
14
|
-
]
|
15
|
-
|
16
|
-
# aggresive test
|
17
|
-
# indentation, comment, capitalize_regex_backreference, regex with space aside.
|
18
|
-
# [DEPLICATED] Use ^....$ pattern for partial word match instead of double-quote-delimiter.
|
19
|
-
CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION = %[
|
20
|
-
capitalize_regex_backreference yes
|
21
|
-
rewriterule1 domain ^www\.google\.com$ site.Google # some comment
|
22
|
-
rewriterule2 domain ^(news)\.(google)\.com$ site.$2$1
|
23
|
-
rewriterule3 agent ^.* Mac OS X .*$ agent.MacOSX
|
24
|
-
rewriterule4 agent "(Googlebot|CustomBot)-([a-zA-Z]+)" agent.$1-$2
|
25
|
-
]
|
26
|
-
|
27
|
-
# remove_tag_prefix test
|
28
|
-
CONFIG_REMOVE_TAG_PREFIX = %[
|
29
|
-
rewriterule1 domain ^www\.google\.com$ ${tag}
|
30
|
-
remove_tag_prefix input
|
31
|
-
]
|
32
|
-
|
33
|
-
# remove_tag_prefix test2
|
34
|
-
CONFIG_REMOVE_TAG_PREFIX_WITH_DOT = %[
|
35
|
-
rewriterule1 domain ^www\.google\.com$ ${tag}
|
36
|
-
remove_tag_prefix input.
|
37
|
-
]
|
38
|
-
|
39
|
-
# hostname placeholder test
|
40
|
-
CONFIG_SHORT_HOSTNAME = %[
|
41
|
-
rewriterule1 domain ^www\.google\.com$ ${hostname}
|
42
|
-
remove_tag_prefix input
|
43
|
-
hostname_command hostname -s
|
44
|
-
]
|
45
|
-
|
46
|
-
# '!' character (exclamation mark) to specify a non-matching pattern
|
47
|
-
CONFIG_NON_MATCHING = %[
|
48
|
-
rewriterule1 domain !^www\..+$ not_start_with_www
|
49
|
-
rewriterule2 domain ^www\..+$ start_with_www
|
50
|
-
]
|
51
|
-
|
52
|
-
# jump of index
|
53
|
-
CONFIG_JUMP_INDEX = %[
|
54
|
-
rewriterule10 domain ^www\.google\.com$ site.Google
|
55
|
-
rewriterule20 domain ^news\.google\.com$ site.GoogleNews
|
56
|
-
]
|
57
|
-
|
58
|
-
# split by tag
|
59
|
-
CONFIG_SPLIT_BY_TAG = %[
|
60
|
-
rewriterule1 user_name ^Lynn Minmay$ vip.${tag_parts[1]}.remember_love
|
61
|
-
rewriterule2 user_name ^Harlock$ ${tag_parts[2]}.${tag_parts[0]}.${tag_parts[1]}
|
62
|
-
rewriterule3 world ^(alice|chaos)$ application.${tag_parts[0]}.$1_server
|
63
|
-
rewriterule4 world ^[a-z]+$ application.${tag_parts[1]}.future_server
|
64
|
-
]
|
65
|
-
|
66
|
-
# test for invalid byte sequence in UTF-8 error
|
67
|
-
CONFIG_INVALID_BYTE = %[
|
68
|
-
rewriterule1 client_name (.+) app.$1
|
69
|
-
]
|
70
|
-
|
71
|
-
def create_driver(conf = CONFIG)
|
8
|
+
def create_driver(conf)
|
72
9
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::RewriteTagFilterOutput).configure(conf)
|
73
10
|
end
|
74
11
|
|
75
12
|
sub_test_case "configure" do
|
76
13
|
data("empty" => "",
|
77
|
-
"
|
78
|
-
"not regexp 1" => "rewriterule1 hoge hoge.${tag_parts[0..2]}.__TAG_PARTS[0..2]__",
|
79
|
-
"not regexp 2" => "rewriterule1 fuga fuga.${tag_parts[1...2]}.__TAG_PARTS[1...2]__")
|
14
|
+
"line style" => "rewriterule1 foo ^foo$ new_tag")
|
80
15
|
test "invalid" do |conf|
|
81
16
|
assert_raise(Fluent::ConfigError) do
|
82
17
|
create_driver(conf)
|
83
18
|
end
|
84
19
|
end
|
85
|
-
|
86
|
-
test "valid" do
|
87
|
-
d = create_driver %[
|
88
|
-
rewriterule1 domain ^www.google.com$ site.Google
|
89
|
-
rewriterule2 domain ^news.google.com$ site.GoogleNews
|
90
|
-
]
|
91
|
-
assert_equal 'domain ^www.google.com$ site.Google', d.instance.config['rewriterule1']
|
92
|
-
assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.config['rewriterule2']
|
93
|
-
end
|
94
20
|
end
|
95
21
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
22
|
+
sub_test_case "section style" do
|
23
|
+
test "simple" do
|
24
|
+
config = %[
|
25
|
+
<rule>
|
26
|
+
key domain
|
27
|
+
pattern ^www\.google\.com$
|
28
|
+
tag site.Google
|
29
|
+
</rule>
|
30
|
+
<rule>
|
31
|
+
key domain
|
32
|
+
pattern ^news\.google\.com$
|
33
|
+
tag site.GoogleNews
|
34
|
+
</rule>
|
35
|
+
<rule>
|
36
|
+
key agent
|
37
|
+
pattern .* Mac OS X .*
|
38
|
+
tag agent.MacOSX
|
39
|
+
</rule>
|
40
|
+
<rule>
|
41
|
+
key agent
|
42
|
+
pattern (Googlebot|CustomBot)-([a-zA-Z]+)
|
43
|
+
tag agent.$1-$2
|
44
|
+
</rule>
|
45
|
+
<rule>
|
46
|
+
key domain
|
47
|
+
pattern ^(tagtest)\.google\.com$
|
48
|
+
tag site.${tag}.$1
|
49
|
+
</rule>
|
50
|
+
]
|
51
|
+
d = create_driver(config)
|
52
|
+
d.run(default_tag: "input.access") do
|
53
|
+
d.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
54
|
+
d.feed({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
|
55
|
+
d.feed({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
|
56
|
+
d.feed({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
|
57
|
+
d.feed({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
|
58
|
+
d.feed({'domain' => 'noop.example.com'}) # to be ignored
|
59
|
+
end
|
60
|
+
events = d.events
|
61
|
+
assert_equal 5, events.length
|
62
|
+
assert_equal 'site.Google', events[0][0] # tag
|
63
|
+
assert_equal 'site.GoogleNews', events[1][0] # tag
|
64
|
+
assert_equal 'news.google.com', events[1][2]['domain']
|
65
|
+
assert_equal 'agent.MacOSX', events[2][0] #tag
|
66
|
+
assert_equal 'agent.Googlebot-FooBar', events[3][0] #tag
|
67
|
+
assert_equal 'site.input.access.tagtest', events[4][0] #tag
|
105
68
|
end
|
106
|
-
events = d1.events
|
107
|
-
assert_equal 5, events.length
|
108
|
-
assert_equal 'site.Google', events[0][0] # tag
|
109
|
-
assert_equal 'site.GoogleNews', events[1][0] # tag
|
110
|
-
assert_equal 'news.google.com', events[1][2]['domain']
|
111
|
-
assert_equal 'agent.MacOSX', events[2][0] #tag
|
112
|
-
assert_equal 'agent.Googlebot-FooBar', events[3][0] #tag
|
113
|
-
assert_equal 'site.input.access.tagtest', events[4][0] #tag
|
114
|
-
end
|
115
69
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
70
|
+
test "remove_tag_prefix" do
|
71
|
+
config = %[
|
72
|
+
remove_tag_prefix input
|
73
|
+
<rule>
|
74
|
+
key domain
|
75
|
+
pattern ^www\.google\.com$
|
76
|
+
tag ${tag}
|
77
|
+
</rule>
|
78
|
+
]
|
79
|
+
d = create_driver(config)
|
80
|
+
d.run(default_tag: "input.access") do
|
81
|
+
d.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
82
|
+
end
|
83
|
+
events = d.events
|
84
|
+
assert_equal 1, events.length
|
85
|
+
assert_equal 'access', events[0][0] # tag
|
123
86
|
end
|
124
|
-
events = d1.events
|
125
|
-
assert_equal 4, events.length
|
126
|
-
assert_equal 'site.Google', events[0][0] # tag
|
127
|
-
assert_equal 'site.GoogleNews', events[1][0] # tag
|
128
|
-
assert_equal 'news.google.com', events[1][2]['domain']
|
129
|
-
assert_equal 'agent.MacOSX', events[2][0] #tag
|
130
|
-
assert_equal 'agent.Googlebot-Foobar', events[3][0] #tag
|
131
|
-
end
|
132
87
|
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
88
|
+
test "remove_tag_prefix with dot" do
|
89
|
+
config = %[
|
90
|
+
remove_tag_prefix input.
|
91
|
+
<rule>
|
92
|
+
key domain
|
93
|
+
pattern ^www\.google\.com$
|
94
|
+
tag ${tag}
|
95
|
+
</rule>
|
96
|
+
]
|
97
|
+
d = create_driver(config)
|
98
|
+
d.run(default_tag: "input.access") do
|
99
|
+
d.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
100
|
+
end
|
101
|
+
events = d.events
|
102
|
+
assert_equal 1, events.length
|
103
|
+
assert_equal 'access', events[0][0] # tag
|
137
104
|
end
|
138
|
-
events = d1.events
|
139
|
-
assert_equal 1, events.length
|
140
|
-
assert_equal 'access', events[0][0] # tag
|
141
|
-
end
|
142
105
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
106
|
+
test "short hostname" do
|
107
|
+
config = %[
|
108
|
+
remove_tag_prefix input
|
109
|
+
hostname_command hostname -s
|
110
|
+
<rule>
|
111
|
+
key domain
|
112
|
+
pattern ^www\.google\.com$
|
113
|
+
tag ${hostname}
|
114
|
+
</rule>
|
115
|
+
]
|
116
|
+
d = create_driver(config)
|
117
|
+
d.run(default_tag: "input.access") do
|
118
|
+
d.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
|
119
|
+
end
|
120
|
+
events = d.events
|
121
|
+
assert_equal 1, events.length
|
122
|
+
assert_equal `hostname -s`.chomp, events[0][0] # tag
|
147
123
|
end
|
148
|
-
events = d1.events
|
149
|
-
assert_equal 1, events.length
|
150
|
-
assert_equal 'access', events[0][0] # tag
|
151
|
-
end
|
152
124
|
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
125
|
+
test "non matching" do
|
126
|
+
config = %[
|
127
|
+
<rule>
|
128
|
+
key domain
|
129
|
+
pattern ^www\..+$
|
130
|
+
tag not_start_with_www
|
131
|
+
invert true
|
132
|
+
</rule>
|
133
|
+
<rule>
|
134
|
+
key domain
|
135
|
+
pattern ^www\..+$
|
136
|
+
tag start_with_www
|
137
|
+
</rule>
|
138
|
+
]
|
139
|
+
d = create_driver(config)
|
140
|
+
d.run(default_tag: "input.access") do
|
141
|
+
d.feed({'domain' => 'www.google.com'})
|
142
|
+
d.feed({'path' => '/'})
|
143
|
+
d.feed({'domain' => 'maps.google.com'})
|
144
|
+
end
|
145
|
+
events = d.events
|
146
|
+
assert_equal 3, events.length
|
147
|
+
assert_equal 'start_with_www', events[0][0] # tag
|
148
|
+
assert_equal 'not_start_with_www', events[1][0] # tag
|
149
|
+
assert_equal 'not_start_with_www', events[2][0] # tag
|
157
150
|
end
|
158
|
-
events = d1.events
|
159
|
-
assert_equal 1, events.length
|
160
|
-
assert_equal `hostname -s`.chomp, events[0][0] # tag
|
161
|
-
end
|
162
151
|
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
152
|
+
test "split by tag" do
|
153
|
+
config = %[
|
154
|
+
<rule>
|
155
|
+
key user_name
|
156
|
+
pattern ^Lynn Minmay$
|
157
|
+
tag vip.${tag_parts[1]}.remember_love
|
158
|
+
</rule>
|
159
|
+
<rule>
|
160
|
+
key user_name
|
161
|
+
pattern ^Harlock$
|
162
|
+
tag ${tag_parts[2]}.${tag_parts[0]}.${tag_parts[1]}
|
163
|
+
</rule>
|
164
|
+
<rule>
|
165
|
+
key world
|
166
|
+
pattern ^(alice|chaos)$
|
167
|
+
tag application.${tag_parts[0]}.$1_server
|
168
|
+
</rule>
|
169
|
+
<rule>
|
170
|
+
key world
|
171
|
+
pattern ^[a-z]+$
|
172
|
+
tag application.${tag_parts[1]}.future_server
|
173
|
+
</rule>
|
174
|
+
]
|
175
|
+
d = create_driver(config)
|
176
|
+
d.run(default_tag: "game.production.api") do
|
177
|
+
d.feed({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
|
178
|
+
d.feed({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
|
179
|
+
d.feed({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
|
180
|
+
d.feed({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
|
181
|
+
d.feed({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
|
182
|
+
end
|
183
|
+
events = d.events
|
184
|
+
assert_equal 5, events.length
|
185
|
+
assert_equal 'application.game.chaos_server', events[0][0]
|
186
|
+
assert_equal 'application.game.chaos_server', events[1][0]
|
187
|
+
assert_equal 'application.production.future_server', events[2][0]
|
188
|
+
assert_equal 'vip.production.remember_love', events[3][0]
|
189
|
+
assert_equal 'api.game.production', events[4][0]
|
169
190
|
end
|
170
|
-
events = d1.events
|
171
|
-
assert_equal 3, events.length
|
172
|
-
assert_equal 'start_with_www', events[0][0] # tag
|
173
|
-
assert_equal 'not_start_with_www', events[1][0] # tag
|
174
|
-
assert_equal 'not_start_with_www', events[2][0] # tag
|
175
|
-
end
|
176
191
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
192
|
+
test "invalid_byte (UTF-8)" do
|
193
|
+
config = %[
|
194
|
+
<rule>
|
195
|
+
key client_name
|
196
|
+
pattern (.+)
|
197
|
+
tag app.$1
|
198
|
+
</rule>
|
199
|
+
]
|
200
|
+
invalid_utf8 = "\xff".force_encoding('UTF-8')
|
201
|
+
d = create_driver(config)
|
202
|
+
d.run(default_tag: "input.activity") do
|
203
|
+
d.feed({'client_name' => invalid_utf8})
|
204
|
+
end
|
205
|
+
events = d.events
|
206
|
+
assert_equal 1, events.length
|
207
|
+
assert_equal "app.?", events[0][0]
|
208
|
+
assert_equal invalid_utf8, events[0][2]['client_name']
|
182
209
|
end
|
183
|
-
events = d1.events
|
184
|
-
assert_equal 2, events.length
|
185
|
-
assert_equal 'site.Google', events[0][0] # tag
|
186
|
-
assert_equal 'site.GoogleNews', events[1][0] # tag
|
187
|
-
end
|
188
210
|
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
211
|
+
test "invalid byte (US-ASCII)" do
|
212
|
+
config = %[
|
213
|
+
<rule>
|
214
|
+
key client_name
|
215
|
+
pattern (.+)
|
216
|
+
tag app.$1
|
217
|
+
</rule>
|
218
|
+
]
|
219
|
+
invalid_ascii = "\xff".force_encoding('US-ASCII')
|
220
|
+
d = create_driver(config)
|
221
|
+
d.run(default_tag: "input.activity") do
|
222
|
+
d.feed({'client_name' => invalid_ascii})
|
223
|
+
end
|
224
|
+
events = d.events
|
225
|
+
assert_equal 1, events.length
|
226
|
+
assert_equal "app.?", events[0][0]
|
227
|
+
assert_equal invalid_ascii, events[0][2]['client_name']
|
197
228
|
end
|
198
|
-
events = d1.events
|
199
|
-
assert_equal 5, events.length
|
200
|
-
assert_equal 'application.game.chaos_server', events[0][0]
|
201
|
-
assert_equal 'application.game.chaos_server', events[1][0]
|
202
|
-
assert_equal 'application.production.future_server', events[2][0]
|
203
|
-
assert_equal 'vip.production.remember_love', events[3][0]
|
204
|
-
assert_equal 'api.game.production', events[4][0]
|
205
|
-
end
|
206
229
|
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
230
|
+
test "nested key support with dot notation" do
|
231
|
+
conf = %[
|
232
|
+
<rule>
|
233
|
+
key $.email.domain
|
234
|
+
pattern ^(example)\.(com)$
|
235
|
+
tag $2.$1
|
236
|
+
</rule>
|
237
|
+
]
|
238
|
+
d = create_driver(conf)
|
239
|
+
d.run(default_tag: "input") do
|
240
|
+
d.feed({ "email" => { "localpart" => "john", "domain" => "example.com" }})
|
241
|
+
d.feed({ "email" => { "localpart" => "doe", "domain" => "example.jp" }})
|
242
|
+
end
|
243
|
+
events = d.events
|
244
|
+
assert_equal "com.example", events[0][0]
|
212
245
|
end
|
213
|
-
events = d1.events
|
214
|
-
assert_equal 1, events.length
|
215
|
-
assert_equal "app.?", events[0][0]
|
216
|
-
assert_equal invalid_utf8, events[0][2]['client_name']
|
217
246
|
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
247
|
+
test "nested key support with bracket notation" do
|
248
|
+
conf = %[
|
249
|
+
<rule>
|
250
|
+
key $['email']['domain']
|
251
|
+
pattern ^(example)\.(com)$
|
252
|
+
tag $2.$1
|
253
|
+
</rule>
|
254
|
+
]
|
255
|
+
d = create_driver(conf)
|
256
|
+
d.run(default_tag: "input") do
|
257
|
+
d.feed({ "email" => { "localpart" => "john", "domain" => "example.com" }})
|
258
|
+
d.feed({ "email" => { "localpart" => "doe", "domain" => "example.jp" }})
|
259
|
+
end
|
260
|
+
events = d.events
|
261
|
+
assert_equal "com.example", events[0][0]
|
222
262
|
end
|
223
|
-
events = d1.events
|
224
|
-
assert_equal 1, events.length
|
225
|
-
assert_equal "app.?", events[0][0]
|
226
|
-
assert_equal invalid_ascii, events[0][2]['client_name']
|
227
263
|
end
|
228
264
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rewrite-tag-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.0
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kentaro Yoshida
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -86,9 +86,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
86
86
|
version: '0'
|
87
87
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
88
88
|
requirements:
|
89
|
-
- - "
|
89
|
+
- - ">="
|
90
90
|
- !ruby/object:Gem::Version
|
91
|
-
version:
|
91
|
+
version: '0'
|
92
92
|
requirements: []
|
93
93
|
rubyforge_project:
|
94
94
|
rubygems_version: 2.6.13
|