fluent-plugin-anonymizer 0.4.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.travis.yml +16 -0
- data/Appraisals +7 -0
- data/README.md +57 -48
- data/fluent-plugin-anonymizer.gemspec +2 -1
- data/gemfiles/fluentd_v0.10.gemfile +7 -0
- data/gemfiles/fluentd_v0.12.gemfile +7 -0
- data/lib/fluent/plugin/anonymizer.rb +1 -1
- data/lib/fluent/plugin/filter_anonymizer.rb +281 -9
- data/lib/fluent/plugin/out_anonymizer.rb +11 -4
- data/test/helper.rb +4 -1
- data/test/plugin/test_filter_anonymizer.rb +146 -13
- data/test/plugin/test_out_anonymizer.rb +14 -14
- metadata +36 -31
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a438e2acef3632942d02792ba4fbe2b877ab90ad
|
4
|
+
data.tar.gz: 8979758d1e489f0bc103d881cacbbfc0b29a326b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a4c53bc239db20b8ec60662476744adc1a28beddd7d300c6dde5a5ab68f63d3892ed61348377592b84ee310988f7136d74395981ead840ea7d6eee0f61256445
|
7
|
+
data.tar.gz: bb927b18908c72e6d0fcc0ea6d1fc9f9f7c1a395d1d8ea0867180dd744b7851d419e25568d88389a8274f672a20faa6a4194e4825b7e0c84334669c080c40093
|
data/.travis.yml
CHANGED
@@ -1,7 +1,23 @@
|
|
1
|
+
sudo: false
|
1
2
|
language: ruby
|
2
3
|
|
3
4
|
rvm:
|
5
|
+
- 2.3.0
|
4
6
|
- 2.2
|
5
7
|
- 2.1
|
6
8
|
- 2.0.0
|
7
9
|
- 1.9.3
|
10
|
+
|
11
|
+
# to avoid travis-ci issue since 2015-12-25
|
12
|
+
before_install:
|
13
|
+
- gem update bundler
|
14
|
+
|
15
|
+
gemfile:
|
16
|
+
- Gemfile
|
17
|
+
- gemfiles/fluentd_v0.10.gemfile
|
18
|
+
- gemfiles/fluentd_v0.12.gemfile
|
19
|
+
|
20
|
+
matrix:
|
21
|
+
allow_failures:
|
22
|
+
- rvm: 1.9.3
|
23
|
+
- rvm: 2.0.0
|
data/Appraisals
ADDED
data/README.md
CHANGED
@@ -2,37 +2,42 @@
|
|
2
2
|
|
3
3
|
## Overview
|
4
4
|
|
5
|
-
Fluentd filter output plugin to anonymize records with [OpenSSL::
|
5
|
+
Fluentd filter output plugin to anonymize records with [OpenSSL::Digest](https://docs.ruby-lang.org/ja/latest/class/OpenSSL=3a=3aDigest.html) of MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as UserID, Email, Phone number, IPv4/IPv6 address and so on.
|
6
6
|
|
7
7
|
## Installation
|
8
8
|
|
9
9
|
install with gem or fluent-gem command as:
|
10
10
|
|
11
11
|
`````
|
12
|
-
|
13
|
-
gem install fluent-plugin-anonymizer
|
12
|
+
# for fluentd
|
13
|
+
$ gem install fluent-plugin-anonymizer
|
14
14
|
|
15
|
-
|
16
|
-
/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-anonymizer
|
15
|
+
# for td-agent
|
16
|
+
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-anonymizer -v 0.5.0
|
17
|
+
|
18
|
+
# for td-agent2
|
19
|
+
$ sudo td-agent-gem install fluent-plugin-anonymizer -v 0.5.0
|
17
20
|
`````
|
18
21
|
|
19
22
|
## Tutorial
|
20
23
|
|
21
|
-
###
|
24
|
+
### Filter Plugin
|
22
25
|
|
23
26
|
#### configuration
|
24
27
|
|
25
|
-
|
26
|
-
|
27
|
-
`````
|
28
|
+
```text
|
28
29
|
<source>
|
29
|
-
type
|
30
|
-
|
30
|
+
@type dummy
|
31
|
+
tag raw.dummy
|
32
|
+
dummy [
|
33
|
+
{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"},
|
34
|
+
{"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
|
35
|
+
]
|
31
36
|
</source>
|
32
37
|
|
33
|
-
<
|
34
|
-
type anonymizer
|
35
|
-
|
38
|
+
<filter raw.**>
|
39
|
+
@type anonymizer
|
40
|
+
|
36
41
|
# Specify hashing keys with comma
|
37
42
|
sha1_keys user_id, member_id, mail
|
38
43
|
|
@@ -43,45 +48,40 @@ It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. F
|
|
43
48
|
ipaddr_mask_keys host
|
44
49
|
ipv4_mask_subnet 24
|
45
50
|
ipv6_mask_subnet 104
|
46
|
-
|
47
|
-
# Set tag rename pattern
|
48
|
-
remove_tag_prefix test.
|
49
|
-
add_tag_prefix anonymized.
|
50
|
-
</match>
|
51
|
+
</filter>
|
51
52
|
|
52
|
-
<match
|
53
|
-
type stdout
|
53
|
+
<match raw.**>
|
54
|
+
@type stdout
|
54
55
|
</match>
|
55
|
-
|
56
|
+
```
|
56
57
|
|
57
58
|
#### result
|
58
59
|
|
59
|
-
|
60
|
-
$ echo '{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
|
61
|
-
$ echo '{"host":"2001:db8:0:8d3:0:8a2e:70:7344","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
|
60
|
+
This sample result has made with the above configuration into "fluent.conf".
|
62
61
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
62
|
+
```text
|
63
|
+
$ fluentd -c fluent.conf
|
64
|
+
2017-02-27 22:59:18.070132000 +0900 raw.dummy: {"host":"10.102.3.0","member_id":"5ab2cebb0537866c4a0cd2e2f3502c0976b788da","mail":"7e9d6dbefa72d56056c8c740b34b5c0bbfec8d87"}
|
65
|
+
2017-02-27 22:59:19.079251000 +0900 raw.dummy: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"445514dfcd82b2a8b94ec6763afa6e349e78c5f8","mail":"54608576c8d815a4ffd595a3c1fe72751ed04424"}
|
66
|
+
2017-02-27 22:59:20.086747000 +0900 raw.dummy: {"host":"10.102.3.0","member_id":"b14a8f98019ec84c6fe329d5af62c46bb45348f8","mail":"723da8084da3438d9287b44e5a714b70e10a9755"}
|
67
|
+
2017-02-27 22:59:21.094767000 +0900 raw.dummy: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"d38ebb9b96c0cbffd4136935c7f6fe9dd05980cd","mail":"b6f9d777831cbecfd2ea806f5f62f79a275bbb82"}
|
68
|
+
```
|
67
69
|
|
68
|
-
###
|
70
|
+
### Output Plugin
|
69
71
|
|
70
72
|
#### configuration
|
71
73
|
|
72
|
-
|
74
|
+
It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. For IP address, auto-detecting IPv4/IPv6 and rounding number with 24bit(IPv4) or 104bit(IPv6) netmask using `ipaddr_mask_keys` and `ipv4_mask_subnet`, `ipv6_mask_subnet` option.
|
75
|
+
|
76
|
+
`````
|
73
77
|
<source>
|
74
|
-
type
|
75
|
-
|
76
|
-
dummy [
|
77
|
-
{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"},
|
78
|
-
{"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
|
79
|
-
]
|
78
|
+
@type forward
|
79
|
+
port 24224
|
80
80
|
</source>
|
81
81
|
|
82
|
-
<
|
83
|
-
type anonymizer
|
84
|
-
|
82
|
+
<match test.message>
|
83
|
+
@type anonymizer
|
84
|
+
|
85
85
|
# Specify hashing keys with comma
|
86
86
|
sha1_keys user_id, member_id, mail
|
87
87
|
|
@@ -92,18 +92,27 @@ $ tail -f /var/log/td-agent/td-agent.log
|
|
92
92
|
ipaddr_mask_keys host
|
93
93
|
ipv4_mask_subnet 24
|
94
94
|
ipv6_mask_subnet 104
|
95
|
-
|
95
|
+
|
96
|
+
# Set tag rename pattern
|
97
|
+
remove_tag_prefix test.
|
98
|
+
add_tag_prefix anonymized.
|
99
|
+
</match>
|
96
100
|
|
97
|
-
<match
|
98
|
-
type stdout
|
101
|
+
<match anonymized.message>
|
102
|
+
@type stdout
|
99
103
|
</match>
|
100
|
-
|
104
|
+
`````
|
101
105
|
|
102
106
|
#### result
|
103
107
|
|
104
|
-
|
105
|
-
$
|
106
|
-
|
108
|
+
`````
|
109
|
+
$ echo '{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
|
110
|
+
$ echo '{"host":"2001:db8:0:8d3:0:8a2e:70:7344","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
|
111
|
+
|
112
|
+
$ tail -f /var/log/td-agent/td-agent.log
|
113
|
+
2014-01-06 18:30:21 +0900 anonymized.message: {"host":"10.102.3.0","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
|
114
|
+
2014-01-06 18:30:22 +0900 anonymized.message: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
|
115
|
+
`````
|
107
116
|
|
108
117
|
## Parameters
|
109
118
|
|
@@ -143,7 +152,7 @@ set one or more option are required for editing tag name using HandleTagNameMixi
|
|
143
152
|
|
144
153
|
* tag
|
145
154
|
|
146
|
-
On using this option [like 'tag anonymized.${tag}' with tag placeholder](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#
|
155
|
+
On using this option [like 'tag anonymized.${tag}' with tag placeholder](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#L179), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
|
147
156
|
|
148
157
|
## Notes
|
149
158
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-anonymizer"
|
7
|
-
spec.version = "0.
|
7
|
+
spec.version = "0.5.0"
|
8
8
|
spec.authors = ["Kentaro Yoshida"]
|
9
9
|
spec.email = ["y.ken.studio@gmail.com"]
|
10
10
|
spec.summary = %q{Fluentd filter output plugin to anonymize records with HMAC of MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as UserID, Email, Phone number, IPv4/IPv6 address and so on.}
|
@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.add_development_dependency "bundler"
|
20
20
|
spec.add_development_dependency "rake"
|
21
21
|
spec.add_development_dependency "test-unit", "~> 3"
|
22
|
+
spec.add_development_dependency "appraisal"
|
22
23
|
spec.add_runtime_dependency "fluentd"
|
23
24
|
spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
|
24
25
|
end
|
@@ -79,7 +79,7 @@ module Fluent
|
|
79
79
|
def anonymize_value(message, algorithm, salt)
|
80
80
|
case algorithm
|
81
81
|
when 'md5','sha1','sha256','sha384','sha512'
|
82
|
-
|
82
|
+
DIGEST[algorithm].call.update(salt).update(message.to_s).hexdigest
|
83
83
|
when 'ipaddr_mask'
|
84
84
|
address = IPAddr.new(message)
|
85
85
|
subnet = address.ipv4? ? @ipv4_mask_subnet : @ipv6_mask_subnet
|
@@ -1,26 +1,298 @@
|
|
1
|
+
require 'fluent/filter'
|
2
|
+
require 'openssl'
|
3
|
+
require 'uri'
|
4
|
+
require 'ipaddr'
|
5
|
+
|
6
|
+
# <filter **>
|
7
|
+
# @type mask
|
8
|
+
# # salts will be selected for field names in a deterministic way
|
9
|
+
# salt secret_salt # different salt for each fields?
|
10
|
+
# salt salt_brabra
|
11
|
+
# salts ["s1","s2","s3","s4"]
|
12
|
+
# <mask sha1>
|
13
|
+
# # key user_id
|
14
|
+
# keys ["user_id","session_id","source_ip"]
|
15
|
+
# key_pattern ^(source|src)_?ip_?(addr|address)?$
|
16
|
+
# value_pattern @mydomain\.example\.com$
|
17
|
+
# value_in_subnet 192.168.0.0/16 # naming?
|
18
|
+
# </mask>
|
19
|
+
# <mask uri_path>
|
20
|
+
# keys ["url","uri"]
|
21
|
+
# # or key_pattern
|
22
|
+
# </mask>
|
23
|
+
# <mask network>
|
24
|
+
# keys ["dest","destination","dest_ip"]
|
25
|
+
# # or key_pattern
|
26
|
+
# ipv4_mask_bits 24
|
27
|
+
# ipv6_mask_bits 104
|
28
|
+
# </mask>
|
29
|
+
# </filter>
|
30
|
+
|
1
31
|
module Fluent
|
2
32
|
class AnonymizerFilter < Filter
|
3
|
-
Plugin.register_filter('anonymizer', self)
|
33
|
+
Fluent::Plugin.register_filter('anonymizer', self)
|
4
34
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
35
|
+
MASK_METHODS = {
|
36
|
+
md5: ->(opts){ ->(v,salt){ OpenSSL::Digest.new("md5").update(salt).update(v.to_s).hexdigest } },
|
37
|
+
sha1: ->(opts){ ->(v,salt){ OpenSSL::Digest.new("sha1").update(salt).update(v.to_s).hexdigest } },
|
38
|
+
sha256: ->(opts){ ->(v,salt){ OpenSSL::Digest.new("sha256").update(salt).update(v.to_s).hexdigest } },
|
39
|
+
sha384: ->(opts){ ->(v,salt){ OpenSSL::Digest.new("sha384").update(salt).update(v.to_s).hexdigest } },
|
40
|
+
sha512: ->(opts){ ->(v,salt){ OpenSSL::Digest.new("sha512").update(salt).update(v.to_s).hexdigest } },
|
41
|
+
uri_path: ->(opts){ ->(v,salt){
|
42
|
+
begin
|
43
|
+
uri = URI.parse(v)
|
44
|
+
if uri.absolute?
|
45
|
+
uri.path = '/'
|
46
|
+
uri.user = uri.password = uri.query = uri.fragment = nil
|
47
|
+
end
|
48
|
+
uri.to_s
|
49
|
+
rescue
|
50
|
+
v
|
51
|
+
end
|
52
|
+
} },
|
53
|
+
network: ->(opts){ ->(v,salt){
|
54
|
+
begin
|
55
|
+
addr = IPAddr.new(v)
|
56
|
+
if addr.ipv4? && opts.ipv4_mask_bits
|
57
|
+
addr.mask(opts.ipv4_mask_bits).to_s
|
58
|
+
elsif addr.ipv6? && opts.ipv6_mask_bits
|
59
|
+
addr.mask(opts.ipv6_mask_bits).to_s
|
60
|
+
else
|
61
|
+
addr.to_s
|
62
|
+
end
|
63
|
+
rescue
|
64
|
+
v
|
65
|
+
end
|
66
|
+
} },
|
67
|
+
}
|
9
68
|
|
10
|
-
|
69
|
+
config_param :salt, :string, default: nil
|
70
|
+
config_param :salts, :array, default: nil
|
71
|
+
config_section :mask, param_name: :mask_config_list, required: false, multi: true do
|
72
|
+
config_argument :method, :enum, list: MASK_METHODS.keys
|
73
|
+
config_param :salt, :string, default: nil
|
74
|
+
|
75
|
+
config_param :key, :string, default: nil
|
76
|
+
config_param :keys, :array, default: []
|
77
|
+
config_param :key_chain, :string, default: nil # for.nested.key
|
78
|
+
config_param :key_chains, :array, default: [] # ["for.nested.key","can.be.specified","twice.or.more"]
|
79
|
+
config_param :key_pattern, :string, default: nil
|
80
|
+
config_param :value_pattern, :string, default: nil
|
81
|
+
config_param :value_in_subnet, :string, default: nil # 192.168.0.0/24
|
82
|
+
|
83
|
+
config_param :mask_array_elements, :bool, default: false
|
84
|
+
config_param :ipv4_mask_bits, :integer, default: nil
|
85
|
+
config_param :ipv6_mask_bits, :integer, default: nil
|
86
|
+
end
|
87
|
+
|
88
|
+
# obsolete configuration parameters
|
89
|
+
config_param :md5_keys, :string, default: nil
|
90
|
+
config_param :sha1_keys, :string, default: nil
|
91
|
+
config_param :sha256_keys, :string, default: nil
|
92
|
+
config_param :sha384_keys, :string, default: nil
|
93
|
+
config_param :sha512_keys, :string, default: nil
|
94
|
+
config_param :hash_salt, :string, default: nil
|
95
|
+
config_param :ipaddr_mask_keys, :string, default: nil
|
96
|
+
config_param :ipv4_mask_subnet, :integer, default: 24
|
97
|
+
config_param :ipv6_mask_subnet, :integer, default: 104
|
11
98
|
|
12
99
|
def initialize
|
13
100
|
super
|
14
|
-
|
101
|
+
@salt_list = []
|
102
|
+
@salt_map = {}
|
103
|
+
@conversions = []
|
15
104
|
end
|
16
105
|
|
17
106
|
def configure(conf)
|
18
107
|
super
|
19
|
-
|
108
|
+
|
109
|
+
salt_missing = false
|
110
|
+
|
111
|
+
@salt_list << @salt
|
112
|
+
@salt_list += @salts if @salts
|
113
|
+
|
114
|
+
@masks = []
|
115
|
+
@mask_config_list.each do |c|
|
116
|
+
unless c.salt || @salt_list.size > 0
|
117
|
+
salt_missing = true
|
118
|
+
end
|
119
|
+
|
120
|
+
conv = MASK_METHODS[c.method].call(c)
|
121
|
+
[c.key || nil, *c.keys].compact.each do |key|
|
122
|
+
@masks << masker_for_key(conv, key, c)
|
123
|
+
end
|
124
|
+
[c.key_chain || nil, *c.key_chains].compact.each do |key_chain|
|
125
|
+
@masks << masker_for_key_chain(conv, key_chain.split('.'), c)
|
126
|
+
end
|
127
|
+
@masks << masker_for_key_pattern(conv, c.key_pattern, c) if c.key_pattern
|
128
|
+
@masks << masker_for_value_pattern(conv, c.value_pattern, c) if c.value_pattern
|
129
|
+
@masks << masker_for_value_in_subnet(conv, c.value_in_subnet, c) if c.value_in_subnet
|
130
|
+
end
|
131
|
+
|
132
|
+
# obsolete option handling
|
133
|
+
[[@md5_keys,:md5],[@sha1_keys,:sha1],[@sha256_keys,:sha256],[@sha384_keys,:sha384],[@sha512_keys,:sha512]].each do |param,m|
|
134
|
+
next unless param
|
135
|
+
@salt_list << (@hash_salt || '') if @salt_list.empty? # to suppress ConfigError for salt missing
|
136
|
+
conf = OpenStruct.new
|
137
|
+
conf.salt = @hash_salt || ''
|
138
|
+
conf.mask_array_elements = true
|
139
|
+
conv = MASK_METHODS[m].call(conf)
|
140
|
+
param.split(',').map(&:strip).each do |key|
|
141
|
+
if key.include?('.')
|
142
|
+
@masks << masker_for_key_chain(conv, key.split('.'), conf)
|
143
|
+
else
|
144
|
+
@masks << masker_for_key(conv, key, conf)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
if @ipaddr_mask_keys
|
149
|
+
@salt_list << (@hash_salt || '') if @salt_list.empty? # to suppress ConfigError for salt missing
|
150
|
+
conf = OpenStruct.new
|
151
|
+
conf.salt = @hash_salt || ''
|
152
|
+
conf.mask_array_elements = true
|
153
|
+
conf.ipv4_mask_bits = @ipv4_mask_subnet
|
154
|
+
conf.ipv6_mask_bits = @ipv6_mask_subnet
|
155
|
+
conv = MASK_METHODS[:network].call(conf)
|
156
|
+
@ipaddr_mask_keys.split(',').map(&:strip).each do |key|
|
157
|
+
if key.include?('.')
|
158
|
+
@masks << masker_for_key_chain(conv, key.split('.'), conf)
|
159
|
+
else
|
160
|
+
@masks << masker_for_key(conv, key, conf)
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
if @masks.size < 1
|
166
|
+
raise Fluent::ConfigError, "no anonymizing operations configured"
|
167
|
+
end
|
168
|
+
if salt_missing
|
169
|
+
raise Fluent::ConfigError, "salt (or salts) required, but missing"
|
170
|
+
end
|
20
171
|
end
|
21
172
|
|
22
173
|
def filter(tag, time, record)
|
23
|
-
record
|
174
|
+
record.update(@masks.reduce(record){|r,mask| mask.call(r)})
|
175
|
+
end
|
176
|
+
|
177
|
+
def filter_stream(tag, es)
|
178
|
+
new_es = MultiEventStream.new
|
179
|
+
es.each do |time, record|
|
180
|
+
new_es.add(time, @masks.reduce(record){|r,mask| mask.call(r) })
|
181
|
+
end
|
182
|
+
new_es
|
183
|
+
end
|
184
|
+
|
185
|
+
def salt_determine(key)
|
186
|
+
return @salt_map[key] if @salt_map.has_key?(key)
|
187
|
+
keystr = key.to_s
|
188
|
+
if keystr.empty?
|
189
|
+
@salt_map[key] = @salt_list[0]
|
190
|
+
else
|
191
|
+
@salt_map[key] = @salt_list[(keystr[0].ord + keystr[-1].ord) % @salt_list.size]
|
192
|
+
end
|
193
|
+
@salt_map[key]
|
194
|
+
end
|
195
|
+
|
196
|
+
def mask_value(value, for_each)
|
197
|
+
if for_each && value.is_a?(Array)
|
198
|
+
value.map{|v|
|
199
|
+
yield v
|
200
|
+
}
|
201
|
+
else
|
202
|
+
yield value
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
def masker_for_key(conv, key, opts)
|
207
|
+
for_each = opts.mask_array_elements
|
208
|
+
salt = opts.salt || salt_determine(key)
|
209
|
+
if for_each
|
210
|
+
->(record){
|
211
|
+
begin
|
212
|
+
if record.has_key?(key)
|
213
|
+
record[key] = mask_value(record[key], for_each){|v| conv.call(v, salt) }
|
214
|
+
end
|
215
|
+
rescue => e
|
216
|
+
log.error "unexpected error while masking value", error_class: e.class, error: e.message
|
217
|
+
end
|
218
|
+
record
|
219
|
+
}
|
220
|
+
else
|
221
|
+
->(record){
|
222
|
+
begin
|
223
|
+
if record.has_key?(key)
|
224
|
+
record[key] = conv.call(record[key], salt)
|
225
|
+
end
|
226
|
+
rescue => e
|
227
|
+
log.error "unexpected error while masking value", error_class: e.class, error: e.message
|
228
|
+
end
|
229
|
+
record
|
230
|
+
}
|
231
|
+
end
|
232
|
+
end
|
233
|
+
|
234
|
+
def masker_for_key_chain(conv, key_chain, opts)
|
235
|
+
for_each = opts.mask_array_elements
|
236
|
+
heading = key_chain[0..-2]
|
237
|
+
container_fetcher = ->(record){ heading.reduce(record){|r,c| r && r.has_key?(c) ? r[c] : nil } }
|
238
|
+
tailing = key_chain[-1]
|
239
|
+
->(record){
|
240
|
+
begin
|
241
|
+
container = container_fetcher.call(record)
|
242
|
+
if container && container.has_key?(tailing)
|
243
|
+
container[tailing] = mask_value(container[tailing], for_each){|v| conv.call(v, opts.salt || salt_determine(tailing)) }
|
244
|
+
end
|
245
|
+
rescue => e
|
246
|
+
log.error "unexpected error while masking value", error_class: e.class, error: e.message
|
247
|
+
end
|
248
|
+
record
|
249
|
+
}
|
250
|
+
end
|
251
|
+
|
252
|
+
def masker_for_key_pattern(conv, pattern, opts)
|
253
|
+
for_each = opts.mask_array_elements
|
254
|
+
regexp = Regexp.new(pattern)
|
255
|
+
->(record){
|
256
|
+
begin
|
257
|
+
record.each_pair do |key, value|
|
258
|
+
next unless (regexp =~ key.to_s rescue nil)
|
259
|
+
record[key] = mask_value(record[key], for_each){|v| conv.call(v, opts.salt || salt_determine(key)) }
|
260
|
+
end
|
261
|
+
rescue => e
|
262
|
+
log.error "unexpected error while masking value", error_class: e.class, error: e.message
|
263
|
+
end
|
264
|
+
record
|
265
|
+
}
|
266
|
+
end
|
267
|
+
|
268
|
+
def masker_for_value_pattern(conv, pattern, opts)
|
269
|
+
regexp = Regexp.new(pattern)
|
270
|
+
->(record){
|
271
|
+
begin
|
272
|
+
record.each_pair do |key, value|
|
273
|
+
next unless (regexp =~ value.to_s rescue nil)
|
274
|
+
record[key] = conv.call(value, opts.salt || salt_determine(key))
|
275
|
+
end
|
276
|
+
rescue => e
|
277
|
+
log.error "unexpected error while masking value", error_class: e.class, error: e.message
|
278
|
+
end
|
279
|
+
record
|
280
|
+
}
|
281
|
+
end
|
282
|
+
|
283
|
+
def masker_for_value_in_subnet(conv, network_str, opts)
|
284
|
+
network = IPAddr.new(network_str)
|
285
|
+
->(record){
|
286
|
+
begin
|
287
|
+
record.each_pair do |key, value|
|
288
|
+
next unless (network.include?(value) rescue nil)
|
289
|
+
record[key] = conv.call(value, opts.salt || salt_determine(key))
|
290
|
+
end
|
291
|
+
rescue => e
|
292
|
+
log.error "unexpected error while masking value", error_class: e.class, error: e.message
|
293
|
+
end
|
294
|
+
record
|
295
|
+
}
|
24
296
|
end
|
25
297
|
end
|
26
298
|
end
|
@@ -13,10 +13,17 @@ class Fluent::AnonymizerOutput < Fluent::Output
|
|
13
13
|
define_method("router") { Fluent::Engine }
|
14
14
|
end
|
15
15
|
|
16
|
-
config_param :tag, :string, :default => nil
|
17
|
-
|
18
|
-
config_param :
|
19
|
-
|
16
|
+
config_param :tag, :string, :default => nil,
|
17
|
+
:desc => 'The output tag.'
|
18
|
+
config_param :hash_salt, :string, :default => '',
|
19
|
+
:desc => <<-DESC
|
20
|
+
This salt affects for md5_keys sha1_keys sha256_keys sha384_keys sha512_keys settings.
|
21
|
+
It is recommend to set a hash salt to prevent rainbow table attacks.
|
22
|
+
DESC
|
23
|
+
config_param :ipv4_mask_subnet, :integer, :default => 24,
|
24
|
+
:desc => 'Anonymize ipv4 addresses by subnet mask.'
|
25
|
+
config_param :ipv6_mask_subnet, :integer, :default => 104,
|
26
|
+
:desc => 'Anonymize ipv6 addresses by subnet mask.'
|
20
27
|
|
21
28
|
include Fluent::HandleTagNameMixin
|
22
29
|
include Fluent::Mixin::RewriteTagName
|
data/test/helper.rb
CHANGED
@@ -23,7 +23,10 @@ unless ENV.has_key?('VERBOSE')
|
|
23
23
|
end
|
24
24
|
|
25
25
|
require 'fluent/plugin/out_anonymizer'
|
26
|
-
|
26
|
+
if Fluent.const_defined?(:Filter)
|
27
|
+
require 'fluent/plugin/filter_anonymizer'
|
28
|
+
end
|
29
|
+
|
27
30
|
|
28
31
|
class Test::Unit::TestCase
|
29
32
|
end
|
@@ -2,6 +2,7 @@ require 'helper'
|
|
2
2
|
|
3
3
|
class AnonymizerFilterTest < Test::Unit::TestCase
|
4
4
|
def setup
|
5
|
+
omit_unless(Fluent.const_defined?(:Filter))
|
5
6
|
Fluent::Test.setup
|
6
7
|
@time = Fluent::Engine.now
|
7
8
|
end
|
@@ -25,13 +26,60 @@ class AnonymizerFilterTest < Test::Unit::TestCase
|
|
25
26
|
d = create_driver(conf)
|
26
27
|
d.run {
|
27
28
|
messages.each {|message|
|
28
|
-
d.
|
29
|
+
d.emit(message, @time)
|
29
30
|
}
|
30
31
|
}
|
31
32
|
filtered = d.filtered_as_array
|
32
33
|
filtered.map {|m| m[2] }
|
33
34
|
end
|
34
35
|
|
36
|
+
require 'ostruct'
|
37
|
+
test 'method md5 works correctly' do
|
38
|
+
conv = Fluent::AnonymizerFilter::MASK_METHODS[:md5].call(OpenStruct.new)
|
39
|
+
digest = conv.call('value1', 'salt')
|
40
|
+
assert_equal 'd21fe9523421f12daad064fd082913fd', digest
|
41
|
+
end
|
42
|
+
test 'method sha1 works correctly' do
|
43
|
+
conv = Fluent::AnonymizerFilter::MASK_METHODS[:sha1].call(OpenStruct.new)
|
44
|
+
digest = conv.call('value2', 'salt')
|
45
|
+
assert_equal 'd2ed8e797065322371012fd8c1a39682987ddb71', digest
|
46
|
+
end
|
47
|
+
test 'method sha256 works correctly' do
|
48
|
+
conv = Fluent::AnonymizerFilter::MASK_METHODS[:sha256].call(OpenStruct.new)
|
49
|
+
digest = conv.call('value3', 'salt')
|
50
|
+
assert_equal 'd70daf9654b8a3ba335f8f9f9638a93e8eba6763a0012ac44a928857871abe82', digest
|
51
|
+
end
|
52
|
+
test 'method sha384 works correctly' do
|
53
|
+
conv = Fluent::AnonymizerFilter::MASK_METHODS[:sha384].call(OpenStruct.new)
|
54
|
+
digest = conv.call('value4', 'salt')
|
55
|
+
assert_equal '646192f8b1ea905238df589a00a10598a53eb245df4ab14b7e9eccf80c37386c99abe5259ccb2ba950003423fa0790ee', digest
|
56
|
+
end
|
57
|
+
test 'method sha512 works correctly' do
|
58
|
+
conv = Fluent::AnonymizerFilter::MASK_METHODS[:sha512].call(OpenStruct.new)
|
59
|
+
digest = conv.call('value5', 'salt')
|
60
|
+
expected = '47c82bfea3783c20e3ba3629f0f827bebf0fa65a9104ada5339e5776e5958f061fe7114bfbe1e9d410aff43c6bee8365adf4fdd072e54ab4fffad820f354f545'
|
61
|
+
assert_equal expected, digest
|
62
|
+
end
|
63
|
+
test 'method uri_path removes path, query parameters, fragment, user and password of uri strings' do
|
64
|
+
conv = Fluent::AnonymizerFilter::MASK_METHODS[:uri_path].call(OpenStruct.new)
|
65
|
+
assert_equal '/my/path', conv.call('/my/path', '')
|
66
|
+
assert_equal 'yay/unknown/format', conv.call('yay/unknown/format', '')
|
67
|
+
assert_equal 'http://example.com/', conv.call('http://example.com/path/to/secret', '')
|
68
|
+
assert_equal 'http://example.com/', conv.call('http://example.com/path/to/secret?a=b', '')
|
69
|
+
assert_equal 'http://example.com/', conv.call('http://example.com/path/to/secret#xxx', '')
|
70
|
+
assert_equal 'http://example.com/', conv.call('http://example.com/path/to/secret?a=b#xxx', '')
|
71
|
+
assert_equal 'http://example.com/', conv.call('http://tagomoris:secret!@example.com/', '')
|
72
|
+
assert_equal 'http://example.com/', conv.call('http://tagomoris:secret!@example.com/?a=b#xxx', '')
|
73
|
+
assert_equal 'http://example.com/', conv.call('http://tagomoris:secret!@example.com/path/to/secret?a=b#xxx', '')
|
74
|
+
end
|
75
|
+
test 'method network masks ipaddresses with specified mask bit lengths' do
|
76
|
+
conf = OpenStruct.new(ipv4_mask_bits: 24, ipv6_mask_bits: 104)
|
77
|
+
conv = Fluent::AnonymizerFilter::MASK_METHODS[:network].call(conf)
|
78
|
+
assert_equal '192.168.1.0', conv.call('192.168.1.1', '')
|
79
|
+
assert_equal '10.110.18.0', conv.call('10.110.18.9', '')
|
80
|
+
assert_equal '2001:db8:0:8d3:0:8a2e::', conv.call('2001:db8:0:8d3:0:8a2e:70:7344', '')
|
81
|
+
end
|
82
|
+
|
35
83
|
def test_configure
|
36
84
|
assert_raise(Fluent::ConfigError) {
|
37
85
|
d = create_driver('')
|
@@ -43,6 +91,91 @@ class AnonymizerFilterTest < Test::Unit::TestCase
|
|
43
91
|
assert_equal 'test_salt_string', d.instance.config['hash_salt']
|
44
92
|
end
|
45
93
|
|
94
|
+
test 'masker_for_key generates a lambda for conversion with exact key match' do
|
95
|
+
conf = OpenStruct.new(salt: 's')
|
96
|
+
plugin = create_driver.instance
|
97
|
+
conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
|
98
|
+
masker = plugin.masker_for_key(conv, 'kkk', conf)
|
99
|
+
r = masker.call({"k" => "x", "kk" => "xx", "kkk" => "xxx", "kkkk" => "xxxx"})
|
100
|
+
assert_equal "x", r["k"]
|
101
|
+
assert_equal "xx", r["kk"]
|
102
|
+
assert_equal "XXX:s", r["kkk"]
|
103
|
+
assert_equal "xxxx", r["kkkk"]
|
104
|
+
assert_equal 4, r.size
|
105
|
+
end
|
106
|
+
|
107
|
+
test 'masker_for_key_chain generates a lambda for conversion with recursive key fetching' do
|
108
|
+
conf = OpenStruct.new(salt: 's')
|
109
|
+
plugin = create_driver.instance
|
110
|
+
conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
|
111
|
+
masker = plugin.masker_for_key_chain(conv, 'a.b.c'.split('.'), conf)
|
112
|
+
event = {
|
113
|
+
'a' => {
|
114
|
+
'b' => { 'c' => 'v', 'd' => 'v' }
|
115
|
+
}
|
116
|
+
}
|
117
|
+
r = masker.call(event)
|
118
|
+
assert_equal 'V:s', r['a']['b']['c']
|
119
|
+
assert_equal 'v', r['a']['b']['d']
|
120
|
+
end
|
121
|
+
|
122
|
+
test 'masker_for_key_pattern generates a lambda for conversion with key pattern match' do
|
123
|
+
conf = OpenStruct.new(salt: 's')
|
124
|
+
plugin = create_driver.instance
|
125
|
+
conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
|
126
|
+
masker = plugin.masker_for_key_pattern(conv, '^k+$', conf)
|
127
|
+
r = masker.call({"k" => "x", "kk" => "xx", "kkk" => "xxx", "kkk0" => "xxxx", "f" => "x", "ff" => "xx"})
|
128
|
+
assert_equal "X:s", r["k"]
|
129
|
+
assert_equal "XX:s", r["kk"]
|
130
|
+
assert_equal "XXX:s", r["kkk"]
|
131
|
+
assert_equal "xxxx", r["kkk0"]
|
132
|
+
assert_equal "x", r["f"]
|
133
|
+
assert_equal "xx", r["ff"]
|
134
|
+
assert_equal 6, r.size
|
135
|
+
end
|
136
|
+
|
137
|
+
test 'masker_for_value_pattern' do
|
138
|
+
conf = OpenStruct.new(salt: 's')
|
139
|
+
plugin = create_driver.instance
|
140
|
+
conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
|
141
|
+
masker = plugin.masker_for_value_pattern(conv, '^x+$', conf)
|
142
|
+
r = masker.call({"k" => "x", "kk" => "x0", "kkk" => "xx0", "kkk0" => "xxxx", "f" => "x", "ff" => "xx"})
|
143
|
+
assert_equal "X:s", r["k"]
|
144
|
+
assert_equal "x0", r["kk"]
|
145
|
+
assert_equal "xx0", r["kkk"]
|
146
|
+
assert_equal "XXXX:s", r["kkk0"]
|
147
|
+
assert_equal "X:s", r["f"]
|
148
|
+
assert_equal "XX:s", r["ff"]
|
149
|
+
assert_equal 6, r.size
|
150
|
+
end
|
151
|
+
|
152
|
+
test 'masker_for_value_in_subnet' do
|
153
|
+
conf = OpenStruct.new(salt: 's')
|
154
|
+
plugin = create_driver.instance
|
155
|
+
conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
|
156
|
+
masker = plugin.masker_for_value_in_subnet(conv, '192.168.0.0/16', conf)
|
157
|
+
r = masker.call({"k1" => "x", "k2" => "192.169.1.1", "k3" => "192.168.128.13", "f1" => "x", "f2" => "10.0.12.1", "f3" => "192.168.1.1"})
|
158
|
+
assert_equal "x", r["k1"]
|
159
|
+
assert_equal "192.169.1.1", r["k2"]
|
160
|
+
assert_equal "192.168.128.13:s", r["k3"]
|
161
|
+
assert_equal "x", r["f1"]
|
162
|
+
assert_equal "10.0.12.1", r["f2"]
|
163
|
+
assert_equal "192.168.1.1:s", r["f3"]
|
164
|
+
end
|
165
|
+
|
166
|
+
test 'filter plugin can mask specified fields' do
|
167
|
+
plugin = create_driver(<<-CONF).instance
|
168
|
+
<mask md5>
|
169
|
+
salt testing
|
170
|
+
key test
|
171
|
+
</mask>
|
172
|
+
CONF
|
173
|
+
r = plugin.filter('tag', Time.now.to_i, {"test" => "value", "name" => "fluentd plugin"})
|
174
|
+
assert_equal 2, r.size
|
175
|
+
assert_equal "fluentd plugin", r["name"]
|
176
|
+
assert_equal "6255093f2e4204e24df48ddd7f4a8abe", r["test"]
|
177
|
+
end
|
178
|
+
|
46
179
|
def test_filter
|
47
180
|
messages = [
|
48
181
|
{
|
@@ -56,11 +189,11 @@ class AnonymizerFilterTest < Test::Unit::TestCase
|
|
56
189
|
]
|
57
190
|
expected = {
|
58
191
|
'host' => '10.102.3.0',
|
59
|
-
'data_for_md5' => '
|
60
|
-
'data_for_sha1' => '
|
61
|
-
'data_for_sha256' => '
|
62
|
-
'data_for_sha384' => '
|
63
|
-
'data_for_sha512' => '
|
192
|
+
'data_for_md5' => '9138bd41172f5485f7b6eee3afcd0d62',
|
193
|
+
'data_for_sha1' => 'ee98db51658d38580b1cf788db19ad06e51a32f7',
|
194
|
+
'data_for_sha256' => 'd53d15615b19597b0f95a984a132ed5164ba9676bf3cb28e018d28feaa2ea6fd',
|
195
|
+
'data_for_sha384' => '6e9cd6d84ea371a72148b418f1a8cb2534da114bc2186d36ec6f14fd5c237b6f2e460f409dda89b7e42a14b7da8a8131',
|
196
|
+
'data_for_sha512' => 'adcf4e5d1e52f57f67d8b0cd85051158d7362103d7ed4cb6302445c2708eff4b17cb309cf5d09fd5cf76615c75652bd29d1707ce689a28e8700afd7a7439ef20'
|
64
197
|
}
|
65
198
|
filtered = filter(CONFIG, messages)
|
66
199
|
assert_equal(expected, filtered[0])
|
@@ -85,9 +218,9 @@ class AnonymizerFilterTest < Test::Unit::TestCase
|
|
85
218
|
expected = {
|
86
219
|
'host' => '10.102.0.0',
|
87
220
|
'host2' => '10.102.0.0',
|
88
|
-
'member_id' => '
|
89
|
-
'mail' => '
|
90
|
-
'telephone' => '
|
221
|
+
'member_id' => '8cb2237d0679ca88db6464eac60da96345513964',
|
222
|
+
'mail' => '914fec35ce8bfa1a067581032f26b053591ee38a',
|
223
|
+
'telephone' => 'ce164718b94212332187eb8420903b46b334d609',
|
91
224
|
'action' => 'signup'
|
92
225
|
}
|
93
226
|
filtered = filter(conf, messages)
|
@@ -118,9 +251,9 @@ class AnonymizerFilterTest < Test::Unit::TestCase
|
|
118
251
|
'host1' => '10.102.0.0'
|
119
252
|
},
|
120
253
|
'nested' => {
|
121
|
-
'data' => '
|
254
|
+
'data' => '8cb2237d0679ca88db6464eac60da96345513964',
|
122
255
|
'nested' => {
|
123
|
-
'data' => '
|
256
|
+
'data' => '8cb2237d0679ca88db6464eac60da96345513964'
|
124
257
|
}
|
125
258
|
}
|
126
259
|
}
|
@@ -142,8 +275,8 @@ class AnonymizerFilterTest < Test::Unit::TestCase
|
|
142
275
|
]
|
143
276
|
expected = {
|
144
277
|
'host' => '10.102.3.0',
|
145
|
-
'array' => ["
|
146
|
-
'hash' => '
|
278
|
+
'array' => ["e3cbba8883fe746c6e35783c9404b4bc0c7ee9eb", "a4ac914c09d7c097fe1f4f96b897e625b6922069"],
|
279
|
+
'hash' => '1a1903d78aed9403649d61cb21ba6b489249761b'
|
147
280
|
}
|
148
281
|
filtered = filter(conf, messages)
|
149
282
|
assert_equal(expected, filtered[0])
|
@@ -49,11 +49,11 @@ class AnonymizerOutputTest < Test::Unit::TestCase
|
|
49
49
|
assert_equal 1, emits.length
|
50
50
|
assert_equal 'anonymized.access', emits[0][0] # tag
|
51
51
|
assert_equal '10.102.3.0', emits[0][2]['host']
|
52
|
-
assert_equal '
|
53
|
-
assert_equal '
|
54
|
-
assert_equal '
|
55
|
-
assert_equal '
|
56
|
-
assert_equal '
|
52
|
+
assert_equal '9138bd41172f5485f7b6eee3afcd0d62', emits[0][2]['data_for_md5']
|
53
|
+
assert_equal 'ee98db51658d38580b1cf788db19ad06e51a32f7', emits[0][2]['data_for_sha1']
|
54
|
+
assert_equal 'd53d15615b19597b0f95a984a132ed5164ba9676bf3cb28e018d28feaa2ea6fd', emits[0][2]['data_for_sha256']
|
55
|
+
assert_equal '6e9cd6d84ea371a72148b418f1a8cb2534da114bc2186d36ec6f14fd5c237b6f2e460f409dda89b7e42a14b7da8a8131', emits[0][2]['data_for_sha384']
|
56
|
+
assert_equal 'adcf4e5d1e52f57f67d8b0cd85051158d7362103d7ed4cb6302445c2708eff4b17cb309cf5d09fd5cf76615c75652bd29d1707ce689a28e8700afd7a7439ef20', emits[0][2]['data_for_sha512']
|
57
57
|
end
|
58
58
|
|
59
59
|
def test_emit_multi_keys
|
@@ -79,9 +79,9 @@ class AnonymizerOutputTest < Test::Unit::TestCase
|
|
79
79
|
assert_equal 'anonymized.access', emits[0][0] # tag
|
80
80
|
assert_equal '10.102.0.0', emits[0][2]['host']
|
81
81
|
assert_equal '10.102.0.0', emits[0][2]['host2']
|
82
|
-
assert_equal '
|
83
|
-
assert_equal '
|
84
|
-
assert_equal '
|
82
|
+
assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['member_id']
|
83
|
+
assert_equal '914fec35ce8bfa1a067581032f26b053591ee38a', emits[0][2]['mail']
|
84
|
+
assert_equal 'ce164718b94212332187eb8420903b46b334d609', emits[0][2]['telephone']
|
85
85
|
assert_equal 'signup', emits[0][2]['action']
|
86
86
|
end
|
87
87
|
|
@@ -110,8 +110,8 @@ class AnonymizerOutputTest < Test::Unit::TestCase
|
|
110
110
|
assert_equal 1, emits.length
|
111
111
|
assert_equal 'anonymized.access', emits[0][0] # tag
|
112
112
|
assert_equal '10.102.0.0', emits[0][2]['hosts']['host1']
|
113
|
-
assert_equal '
|
114
|
-
assert_equal '
|
113
|
+
assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['nested']['data']
|
114
|
+
assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['nested']['nested']['data']
|
115
115
|
end
|
116
116
|
|
117
117
|
def test_emit_nest_value
|
@@ -132,8 +132,8 @@ class AnonymizerOutputTest < Test::Unit::TestCase
|
|
132
132
|
assert_equal 1, emits.length
|
133
133
|
assert_equal 'anonymized.access', emits[0][0] # tag
|
134
134
|
assert_equal '10.102.3.0', emits[0][2]['host']
|
135
|
-
assert_equal ["
|
136
|
-
assert_equal '
|
135
|
+
assert_equal ["e3cbba8883fe746c6e35783c9404b4bc0c7ee9eb", "a4ac914c09d7c097fe1f4f96b897e625b6922069"], emits[0][2]['array']
|
136
|
+
assert_equal '1a1903d78aed9403649d61cb21ba6b489249761b', emits[0][2]['hash']
|
137
137
|
end
|
138
138
|
|
139
139
|
def test_emit_ipv6
|
@@ -170,7 +170,7 @@ class AnonymizerOutputTest < Test::Unit::TestCase
|
|
170
170
|
emits = d1.emits
|
171
171
|
assert_equal 1, emits.length
|
172
172
|
assert_equal 'anonymized.message', emits[0][0] # tag
|
173
|
-
assert_equal '
|
173
|
+
assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['member_id']
|
174
174
|
end
|
175
175
|
|
176
176
|
def test_emit_tag_placeholder
|
@@ -187,7 +187,7 @@ class AnonymizerOutputTest < Test::Unit::TestCase
|
|
187
187
|
emits = d1.emits
|
188
188
|
assert_equal 1, emits.length
|
189
189
|
assert_equal 'anonymized.access', emits[0][0] # tag
|
190
|
-
assert_equal '
|
190
|
+
assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['member_id']
|
191
191
|
end
|
192
192
|
end
|
193
193
|
|
metadata
CHANGED
@@ -1,94 +1,97 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-anonymizer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.5.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Kentaro Yoshida
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2017-02-27 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: bundler
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - ">="
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :development
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: rake
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - ">="
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :development
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - ">="
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: test-unit
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- - ~>
|
45
|
+
- - "~>"
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: '3'
|
54
48
|
type: :development
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- - ~>
|
52
|
+
- - "~>"
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: '3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: appraisal
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
62
69
|
- !ruby/object:Gem::Dependency
|
63
70
|
name: fluentd
|
64
71
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
72
|
requirements:
|
67
|
-
- -
|
73
|
+
- - ">="
|
68
74
|
- !ruby/object:Gem::Version
|
69
75
|
version: '0'
|
70
76
|
type: :runtime
|
71
77
|
prerelease: false
|
72
78
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
79
|
requirements:
|
75
|
-
- -
|
80
|
+
- - ">="
|
76
81
|
- !ruby/object:Gem::Version
|
77
82
|
version: '0'
|
78
83
|
- !ruby/object:Gem::Dependency
|
79
84
|
name: fluent-mixin-rewrite-tag-name
|
80
85
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
86
|
requirements:
|
83
|
-
- -
|
87
|
+
- - ">="
|
84
88
|
- !ruby/object:Gem::Version
|
85
89
|
version: '0'
|
86
90
|
type: :runtime
|
87
91
|
prerelease: false
|
88
92
|
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
93
|
requirements:
|
91
|
-
- -
|
94
|
+
- - ">="
|
92
95
|
- !ruby/object:Gem::Version
|
93
96
|
version: '0'
|
94
97
|
description:
|
@@ -98,13 +101,16 @@ executables: []
|
|
98
101
|
extensions: []
|
99
102
|
extra_rdoc_files: []
|
100
103
|
files:
|
101
|
-
- .gitignore
|
102
|
-
- .travis.yml
|
104
|
+
- ".gitignore"
|
105
|
+
- ".travis.yml"
|
106
|
+
- Appraisals
|
103
107
|
- Gemfile
|
104
108
|
- LICENSE
|
105
109
|
- README.md
|
106
110
|
- Rakefile
|
107
111
|
- fluent-plugin-anonymizer.gemspec
|
112
|
+
- gemfiles/fluentd_v0.10.gemfile
|
113
|
+
- gemfiles/fluentd_v0.12.gemfile
|
108
114
|
- lib/fluent/plugin/anonymizer.rb
|
109
115
|
- lib/fluent/plugin/filter_anonymizer.rb
|
110
116
|
- lib/fluent/plugin/out_anonymizer.rb
|
@@ -114,27 +120,26 @@ files:
|
|
114
120
|
homepage: https://github.com/y-ken/fluent-plugin-anonymizer
|
115
121
|
licenses:
|
116
122
|
- Apache License, Version 2.0
|
123
|
+
metadata: {}
|
117
124
|
post_install_message:
|
118
125
|
rdoc_options: []
|
119
126
|
require_paths:
|
120
127
|
- lib
|
121
128
|
required_ruby_version: !ruby/object:Gem::Requirement
|
122
|
-
none: false
|
123
129
|
requirements:
|
124
|
-
- -
|
130
|
+
- - ">="
|
125
131
|
- !ruby/object:Gem::Version
|
126
132
|
version: '0'
|
127
133
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
128
|
-
none: false
|
129
134
|
requirements:
|
130
|
-
- -
|
135
|
+
- - ">="
|
131
136
|
- !ruby/object:Gem::Version
|
132
137
|
version: '0'
|
133
138
|
requirements: []
|
134
139
|
rubyforge_project:
|
135
|
-
rubygems_version:
|
140
|
+
rubygems_version: 2.2.5
|
136
141
|
signing_key:
|
137
|
-
specification_version:
|
142
|
+
specification_version: 4
|
138
143
|
summary: Fluentd filter output plugin to anonymize records with HMAC of MD5/SHA1/SHA256/SHA384/SHA512
|
139
144
|
algorithms. This data masking plugin protects privacy data such as UserID, Email,
|
140
145
|
Phone number, IPv4/IPv6 address and so on.
|