fluent-plugin-td 0.10.9 → 0.10.10
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +7 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_tdlog.rb +85 -0
- metadata +2 -2
data/ChangeLog
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.10.
|
1
|
+
0.10.10
|
@@ -6,6 +6,60 @@ class TreasureDataLogOutput < BufferedOutput
|
|
6
6
|
|
7
7
|
IMPORT_SIZE_LIMIT = 32*1024*1024
|
8
8
|
|
9
|
+
class Anonymizer
|
10
|
+
include Configurable
|
11
|
+
end
|
12
|
+
|
13
|
+
class RawAnonymizer < Anonymizer
|
14
|
+
def anonymize(obj)
|
15
|
+
if obj == nil
|
16
|
+
nil
|
17
|
+
elsif obj.is_a?(String)
|
18
|
+
anonymize_raw obj
|
19
|
+
elsif obj.is_a?(Numeric)
|
20
|
+
anonymize_raw obj.to_s
|
21
|
+
else
|
22
|
+
# boolean, array, map
|
23
|
+
anonymize_raw MessagePack.pack(obj)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class MD5Anonymizer < RawAnonymizer
|
29
|
+
def anonymize_raw(raw)
|
30
|
+
Digest::MD5.hexdigest(raw)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class IPXORAnonymizer < RawAnonymizer
|
35
|
+
config_param :xor_key, :string
|
36
|
+
|
37
|
+
def configure(conf)
|
38
|
+
super
|
39
|
+
|
40
|
+
a1, a2, a3, a4 = @xor_key.split('.')
|
41
|
+
@xor_keys = [a1.to_i, a2.to_i, a3.to_i, a4.to_i]
|
42
|
+
|
43
|
+
if @xor_keys == [0, 0, 0, 0]
|
44
|
+
raise ConfigError, "'xor_key' must be IPv4 address"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def anonymize_raw(raw)
|
49
|
+
m = /\A(\d+)\.(\d+)\.(\d+)\.(\d+)/.match(raw)
|
50
|
+
return nil unless m
|
51
|
+
|
52
|
+
k1, k2, k3, k4 = @xor_keys
|
53
|
+
|
54
|
+
o1 = m[1].to_i ^ k1
|
55
|
+
o2 = m[2].to_i ^ k2
|
56
|
+
o3 = m[3].to_i ^ k3
|
57
|
+
o4 = m[4].to_i ^ k4
|
58
|
+
|
59
|
+
"#{o1}.#{o2}.#{o3}.#{o4}"
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
9
63
|
def initialize
|
10
64
|
require 'fileutils'
|
11
65
|
require 'tempfile'
|
@@ -15,6 +69,7 @@ class TreasureDataLogOutput < BufferedOutput
|
|
15
69
|
require 'cgi' # CGI.escape
|
16
70
|
require 'time' # Time#rfc2822
|
17
71
|
require 'td-client'
|
72
|
+
require 'digest/md5'
|
18
73
|
super
|
19
74
|
@tmpdir = '/tmp/fluent/tdlog'
|
20
75
|
@apikey = nil
|
@@ -90,6 +145,28 @@ class TreasureDataLogOutput < BufferedOutput
|
|
90
145
|
@key = "#{database}.#{table}"
|
91
146
|
end
|
92
147
|
|
148
|
+
@anonymizes = {}
|
149
|
+
conf.elements.select {|e|
|
150
|
+
e.name == 'anonymize'
|
151
|
+
}.each {|e|
|
152
|
+
key = e['key']
|
153
|
+
method = e['method']
|
154
|
+
|
155
|
+
case method
|
156
|
+
when 'md5'
|
157
|
+
scr = MD5Anonymizer.new
|
158
|
+
when 'ip_xor'
|
159
|
+
scr = IPXORAnonymizer.new
|
160
|
+
else
|
161
|
+
raise ConfigError, "Unknown anonymize method: #{method}"
|
162
|
+
end
|
163
|
+
|
164
|
+
scr.configure(e)
|
165
|
+
|
166
|
+
@anonymizes[key] = scr
|
167
|
+
}
|
168
|
+
@anonymizes = nil if @anonymizes.empty?
|
169
|
+
|
93
170
|
@http_proxy = conf['http_proxy']
|
94
171
|
end
|
95
172
|
|
@@ -123,6 +200,14 @@ class TreasureDataLogOutput < BufferedOutput
|
|
123
200
|
off = out.bytesize
|
124
201
|
es.each {|time,record|
|
125
202
|
begin
|
203
|
+
if @anonymizes
|
204
|
+
@anonymizes.each_pair {|key,scr|
|
205
|
+
if value = record[key]
|
206
|
+
record[key] = scr.anonymize(value)
|
207
|
+
end
|
208
|
+
}
|
209
|
+
end
|
210
|
+
|
126
211
|
record['time'] = time
|
127
212
|
|
128
213
|
if record.size > @key_num_limit
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
4
|
+
version: 0.10.10
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-09-25 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|