fluent-plugin-secure-forward 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +24 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/README.md +238 -0
- data/Rakefile +2 -0
- data/example/auth_client.conf +19 -0
- data/example/auth_server.conf +30 -0
- data/example/cert_client.conf +22 -0
- data/example/cert_server.conf +34 -0
- data/example/certs/cert.pem +18 -0
- data/example/certs/key.pem +15 -0
- data/example/client.conf +13 -0
- data/example/server.conf +10 -0
- data/fluent-plugin-secure-forward.gemspec +20 -0
- data/lib/fluent/plugin/in_secure_forward.rb +402 -0
- data/lib/fluent/plugin/out_secure_forward.rb +417 -0
- data/test/plugin/test_in_secure_forward.rb +0 -0
- data/test/plugin/test_out_secure_forward.rb +0 -0
- metadata +129 -0
data/example/server.conf
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
Gem::Specification.new do |gem|
|
3
|
+
gem.name = "fluent-plugin-secure-forward"
|
4
|
+
gem.version = "0.0.1"
|
5
|
+
gem.authors = ["TAGOMORI Satoshi"]
|
6
|
+
gem.email = ["tagomoris@gmail.com"]
|
7
|
+
gem.summary = %q{Fluentd input/output plugin to forward over SSL with authentications}
|
8
|
+
gem.description = %q{This version is HIGHLY EXPERIMENTAL. DON'T USE IN PRODUCTION}
|
9
|
+
gem.homepage = "https://github.com/tagomoris/fluent-plugin-secure-forward"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.require_paths = ["lib"]
|
15
|
+
|
16
|
+
gem.add_development_dependency "fluentd"
|
17
|
+
gem.add_development_dependency "fluent-mixin-config-placeholders"
|
18
|
+
gem.add_runtime_dependency "fluentd"
|
19
|
+
gem.add_runtime_dependency "fluent-mixin-config-placeholders"
|
20
|
+
end
|
@@ -0,0 +1,402 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require 'fluent/mixin/config_placeholders'
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
class SecureForwardInput < Input
|
7
|
+
DEFAULT_SECURE_LISTEN_PORT = 24284
|
8
|
+
|
9
|
+
Fluent::Plugin.register_input('secure_forward', self)
|
10
|
+
|
11
|
+
config_param :self_hostname, :string
|
12
|
+
include Fluent::Mixin::ConfigPlaceholders
|
13
|
+
|
14
|
+
config_param :shared_key, :string
|
15
|
+
|
16
|
+
config_param :bind, :string, :default => '0.0.0.0'
|
17
|
+
config_param :port, :integer, :default => DEFAULT_SECURE_LISTEN_PORT
|
18
|
+
config_param :allow_keepalive, :bool, :default => true #TODO: implement
|
19
|
+
|
20
|
+
config_param :allow_anonymous_source, :bool, :default => true
|
21
|
+
config_param :authentication, :bool, :default => false
|
22
|
+
|
23
|
+
## meaningless for security...? not implemented yet
|
24
|
+
# config_param :dns_reverse_lookup_check, :bool, :default => false
|
25
|
+
|
26
|
+
config_param :cert_auto_generate, :bool, :default => false
|
27
|
+
config_param :generate_private_key_length, :integer, :default => 2048
|
28
|
+
|
29
|
+
config_param :generate_cert_country, :string, :default => 'US'
|
30
|
+
config_param :generate_cert_state, :string, :default => 'CA'
|
31
|
+
config_param :generate_cert_locality, :string, :default => 'Mountain View'
|
32
|
+
config_param :generate_cert_common_name, :string, :default => nil
|
33
|
+
|
34
|
+
config_param :cert_file_path, :string, :default => nil
|
35
|
+
config_param :private_key_file, :string, :default => nil
|
36
|
+
config_param :private_key_passphrase, :string, :default => nil
|
37
|
+
|
38
|
+
config_param :read_length, :size, :default => 8*1024*1024 # 8MB
|
39
|
+
config_param :read_interval_msec, :integer, :default => 50 # 50ms
|
40
|
+
config_param :socket_interval_msec, :integer, :default => 200 # 200ms
|
41
|
+
|
42
|
+
attr_reader :read_interval, :socket_interval
|
43
|
+
|
44
|
+
attr_reader :users # list of (username, password) by <user> tag
|
45
|
+
# <user>
|
46
|
+
# username ....
|
47
|
+
# password ....
|
48
|
+
# </user>
|
49
|
+
attr_reader :nodes # list of hosts, allowed to connect <server> tag (it includes source ip, shared_key(optional))
|
50
|
+
# <client>
|
51
|
+
# host ipaddr/hostname
|
52
|
+
# shared_key .... # optional shared key
|
53
|
+
# users username,list,of,allowed
|
54
|
+
# </client>
|
55
|
+
|
56
|
+
attr_reader :sessions # node/socket/thread list which has sslsocket instance keepaliving to client
|
57
|
+
|
58
|
+
def initialize
|
59
|
+
super
|
60
|
+
require 'resolv'
|
61
|
+
require 'socket'
|
62
|
+
require 'openssl'
|
63
|
+
require 'digest'
|
64
|
+
end
|
65
|
+
|
66
|
+
def configure(conf)
|
67
|
+
super
|
68
|
+
|
69
|
+
unless @cert_auto_generate || @cert_file_path
|
70
|
+
raise Fluent::ConfigError, "One of 'cert_auto_generate' or 'cert_file_path' must be specified"
|
71
|
+
end
|
72
|
+
|
73
|
+
@read_interval = @read_interval_msec / 1000.0
|
74
|
+
@socket_interval = @socket_interval_msec / 1000.0
|
75
|
+
|
76
|
+
@users = []
|
77
|
+
@nodes = []
|
78
|
+
conf.elements.each do |element|
|
79
|
+
case element.name
|
80
|
+
when 'user'
|
81
|
+
unless element['username'] && element['password']
|
82
|
+
raise Fluent::ConfigError, "username/password pair missing in <user>"
|
83
|
+
end
|
84
|
+
@users.push({
|
85
|
+
username: element['username'],
|
86
|
+
password: element['password']
|
87
|
+
})
|
88
|
+
when 'client'
|
89
|
+
unless element['host']
|
90
|
+
raise Fluent::ConfigError, "host missing in <client>"
|
91
|
+
end
|
92
|
+
@nodes.push({
|
93
|
+
host: element['host'],
|
94
|
+
shared_key: (element['shared_key'] || @shared_key),
|
95
|
+
users: (element['users'] ? element['users'].split(',') : nil),
|
96
|
+
})
|
97
|
+
else
|
98
|
+
raise Fluent::ConfigError, "unknown config tag name"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
@generate_cert_common_name ||= @self_hostname
|
103
|
+
self.certificate
|
104
|
+
true
|
105
|
+
end
|
106
|
+
|
107
|
+
def start
|
108
|
+
super
|
109
|
+
OpenSSL::Random.seed(File.read("/dev/random", 16))
|
110
|
+
@sessions = []
|
111
|
+
@sock = nil
|
112
|
+
@listener = Thread.new(&method(:run))
|
113
|
+
end
|
114
|
+
|
115
|
+
def shutdown
|
116
|
+
@listener.kill
|
117
|
+
@listener.join
|
118
|
+
@sessions.each{ |s| s.shutdown }
|
119
|
+
@sock.close
|
120
|
+
end
|
121
|
+
|
122
|
+
def select_authenticate_users(node, username)
|
123
|
+
if node.nil? || node[:users].nil?
|
124
|
+
@users.select{|u| u[:username] == username}
|
125
|
+
else
|
126
|
+
@users.select{|u| node[:users].include?(u[:username]) && u[:username] == username}
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def certificate
|
131
|
+
return @cert, @key if @cert && @key
|
132
|
+
|
133
|
+
if @cert_auto_generate
|
134
|
+
key = OpenSSL::PKey::RSA.generate(@generate_private_key_length)
|
135
|
+
|
136
|
+
digest = OpenSSL::Digest::SHA1.new
|
137
|
+
issuer = subject = OpenSSL::X509::Name.new
|
138
|
+
subject.add_entry('C', @generate_cert_country)
|
139
|
+
subject.add_entry('ST', @generate_cert_state)
|
140
|
+
subject.add_entry('L', @generate_cert_locality)
|
141
|
+
subject.add_entry('CN', @generate_cert_common_name)
|
142
|
+
|
143
|
+
cer = OpenSSL::X509::Certificate.new
|
144
|
+
cer.not_before = Time.at(0)
|
145
|
+
cer.not_after = Time.at(0)
|
146
|
+
cer.public_key = key
|
147
|
+
cer.serial = 1
|
148
|
+
cer.issuer = issuer
|
149
|
+
cer.subject = subject
|
150
|
+
cer.sign(key, digest)
|
151
|
+
|
152
|
+
@cert = cer
|
153
|
+
@key = key
|
154
|
+
return @cert, @key
|
155
|
+
end
|
156
|
+
|
157
|
+
@cert = OpenSSL::X509::Certificate.new(File.read(@cert_file_path))
|
158
|
+
@key = OpenSSL::PKey::RSA.new(File.read(@private_key_file), @private_key_passphrase)
|
159
|
+
end
|
160
|
+
|
161
|
+
def run # sslsocket server thread
|
162
|
+
cert, key = self.certificate
|
163
|
+
ctx = OpenSSL::SSL::SSLContext.new
|
164
|
+
ctx.cert = cert
|
165
|
+
ctx.key = key
|
166
|
+
|
167
|
+
server = TCPServer.new(@bind, @port)
|
168
|
+
@sock = OpenSSL::SSL::SSLServer.new(server, ctx)
|
169
|
+
loop do
|
170
|
+
while socket = @sock.accept
|
171
|
+
@sessions.push Session.new(self, socket)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
def on_message(msg)
|
177
|
+
# NOTE: copy&paste from Fluent::ForwardInput#on_message(msg)
|
178
|
+
|
179
|
+
# TODO: format error
|
180
|
+
tag = msg[0].to_s
|
181
|
+
entries = msg[1]
|
182
|
+
|
183
|
+
if entries.class == String
|
184
|
+
# PackedForward
|
185
|
+
es = MessagePackEventStream.new(entries, @cached_unpacker)
|
186
|
+
Fluent::Engine.emit_stream(tag, es)
|
187
|
+
|
188
|
+
elsif entries.class == Array
|
189
|
+
# Forward
|
190
|
+
es = Fluent::MultiEventStream.new
|
191
|
+
entries.each {|e|
|
192
|
+
time = e[0].to_i
|
193
|
+
time = (now ||= Fluent::Engine.now) if time == 0
|
194
|
+
record = e[1]
|
195
|
+
es.add(time, record)
|
196
|
+
}
|
197
|
+
Fluent::Engine.emit_stream(tag, es)
|
198
|
+
|
199
|
+
else
|
200
|
+
# Message
|
201
|
+
time = msg[1]
|
202
|
+
time = Fluent::Engine.now if time == 0
|
203
|
+
record = msg[2]
|
204
|
+
Fluent::Engine.emit(tag, time, record)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
class Session # Fluent::SecureForwardInput::Session
|
209
|
+
attr_accessor :receiver
|
210
|
+
attr_accessor :state, :thread, :node, :socket, :unpacker, :auth_salt
|
211
|
+
|
212
|
+
def initialize(receiver, socket)
|
213
|
+
@receiver = receiver
|
214
|
+
|
215
|
+
@state = :helo
|
216
|
+
|
217
|
+
@socket = socket
|
218
|
+
@socket.sync = true
|
219
|
+
|
220
|
+
@ipaddress = nil
|
221
|
+
@node = nil
|
222
|
+
@unpacker = MessagePack::Unpacker.new
|
223
|
+
@thread = Thread.new(&method(:start))
|
224
|
+
end
|
225
|
+
|
226
|
+
def established?
|
227
|
+
@state == :established
|
228
|
+
end
|
229
|
+
|
230
|
+
def generate_salt
|
231
|
+
OpenSSL::Random.random_bytes(16)
|
232
|
+
end
|
233
|
+
|
234
|
+
def check_node(hostname, ipaddress, port, proto)
|
235
|
+
node = nil
|
236
|
+
family = Socket.const_get(proto)
|
237
|
+
@receiver.nodes.each do |n|
|
238
|
+
proto, port, host, ipaddr, family_num, socktype_num, proto_num = Socket.getaddrinfo(n[:host], port, family).first
|
239
|
+
if ipaddr == ipaddress
|
240
|
+
node = n
|
241
|
+
break
|
242
|
+
end
|
243
|
+
end
|
244
|
+
node
|
245
|
+
end
|
246
|
+
|
247
|
+
## not implemented yet
|
248
|
+
# def check_hostname_reverse_lookup(ipaddress)
|
249
|
+
# rev_name = Resolv.getname(ipaddress)
|
250
|
+
# proto, port, host, ipaddr, family_num, socktype_num, proto_num = Socket.getaddrinfo(rev_name, DUMMY_PORT)
|
251
|
+
# unless ipaddr == ipaddress
|
252
|
+
# return false
|
253
|
+
# end
|
254
|
+
# true
|
255
|
+
# end
|
256
|
+
|
257
|
+
def generate_helo
|
258
|
+
$log.debug "generating helo"
|
259
|
+
# ['HELO', options(hash)]
|
260
|
+
[ 'HELO', {'auth' => (@receiver.authentication ? @auth_key_salt : ''), 'keepalive' => @receiver.allow_keepalive } ]
|
261
|
+
end
|
262
|
+
|
263
|
+
def check_ping(message)
|
264
|
+
$log.debug "checking ping"
|
265
|
+
# ['PING', self_hostname, shared_key\_salt, sha512\_hex(shared_key\_salt + self_hostname + shared_key),
|
266
|
+
# username || '', sha512\_hex(auth\_salt + username + password) || '']
|
267
|
+
unless message.size == 6 && message[0] == 'PING'
|
268
|
+
return false, 'invalid ping message'
|
269
|
+
end
|
270
|
+
ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
|
271
|
+
|
272
|
+
shared_key = if @node && @node[:shared_key]
|
273
|
+
@node[:shared_key]
|
274
|
+
else
|
275
|
+
@receiver.shared_key
|
276
|
+
end
|
277
|
+
serverside = Digest::SHA512.new.update(shared_key_salt).update(hostname).update(shared_key).hexdigest
|
278
|
+
if shared_key_hexdigest != serverside
|
279
|
+
$log.warn "Shared key mismatch from '#{hostname}'"
|
280
|
+
return false, 'shared_key mismatch'
|
281
|
+
end
|
282
|
+
|
283
|
+
if @receiver.authentication
|
284
|
+
users = @receiver.select_authenticate_users(@node, username)
|
285
|
+
success = false
|
286
|
+
users.each do |user|
|
287
|
+
passhash = Digest::SHA512.new.update(@auth_key_salt).update(username).update(user[:password]).hexdigest
|
288
|
+
success ||= (passhash == password_digest)
|
289
|
+
end
|
290
|
+
unless success
|
291
|
+
$log.warn "Authentication failed from client '#{hostname}', username '#{username}'"
|
292
|
+
return false, 'username/password mismatch'
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
return true, shared_key_salt
|
297
|
+
end
|
298
|
+
|
299
|
+
def generate_pong(auth_result, reason_or_salt)
|
300
|
+
$log.debug "generating pong"
|
301
|
+
# ['PONG', bool(authentication result), 'reason if authentication failed',
|
302
|
+
# self_hostname, sha512\_hex(salt + self_hostname + sharedkey)]
|
303
|
+
if not auth_result
|
304
|
+
return ['PONG', false, reason_or_salt, '', '']
|
305
|
+
end
|
306
|
+
|
307
|
+
shared_key = if @node && @node[:shared_key]
|
308
|
+
@node[:shared_key]
|
309
|
+
else
|
310
|
+
@receiver.shared_key
|
311
|
+
end
|
312
|
+
shared_key_hex = Digest::SHA512.new.update(reason_or_salt).update(@receiver.self_hostname).update(shared_key).hexdigest
|
313
|
+
[ 'PONG', true, '', @receiver.self_hostname, shared_key_hex ]
|
314
|
+
end
|
315
|
+
|
316
|
+
def on_read(data)
|
317
|
+
$log.debug "on_read"
|
318
|
+
if self.established?
|
319
|
+
@receiver.on_message(data)
|
320
|
+
end
|
321
|
+
|
322
|
+
case @state
|
323
|
+
when :pingpong
|
324
|
+
success, reason_or_salt = self.check_ping(data)
|
325
|
+
if not success
|
326
|
+
send_data generate_pong(false, reason_or_salt)
|
327
|
+
self.shutdown
|
328
|
+
return
|
329
|
+
end
|
330
|
+
send_data generate_pong(true, reason_or_salt)
|
331
|
+
|
332
|
+
$log.debug "connection established"
|
333
|
+
@state = :established
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
def send_data(data)
|
338
|
+
# not nonblock because write data (response) needs sequence
|
339
|
+
@socket.write data.to_msgpack
|
340
|
+
end
|
341
|
+
|
342
|
+
def start
|
343
|
+
$log.debug "starting server"
|
344
|
+
|
345
|
+
proto, port, host, ipaddr = @socket.io.addr
|
346
|
+
@node = check_node(host, ipaddr, port, proto)
|
347
|
+
if @node.nil? && (! @receiver.allow_anonymous_source)
|
348
|
+
$log.warn "Connection required from unknown host '#{host}' (#{ipaddr}), disconnecting..."
|
349
|
+
self.shutdown
|
350
|
+
end
|
351
|
+
|
352
|
+
@auth_key_salt = generate_salt
|
353
|
+
|
354
|
+
buf = ''
|
355
|
+
read_length = @receiver.read_length
|
356
|
+
read_interval = @receiver.read_interval
|
357
|
+
socket_interval = @receiver.socket_interval
|
358
|
+
|
359
|
+
send_data generate_helo()
|
360
|
+
@state = :pingpong
|
361
|
+
|
362
|
+
loop do
|
363
|
+
begin
|
364
|
+
while @socket.read_nonblock(read_length, buf)
|
365
|
+
if buf == ''
|
366
|
+
sleep read_interval
|
367
|
+
next
|
368
|
+
end
|
369
|
+
@unpacker.feed_each(buf, &method(:on_read))
|
370
|
+
buf = ''
|
371
|
+
end
|
372
|
+
rescue OpenSSL::SSL::SSLError => e
|
373
|
+
# to wait i/o restart
|
374
|
+
sleep socket_interval
|
375
|
+
rescue EOFError => e
|
376
|
+
$log.debug "Connection closed from '#{host}'(#{ipaddr})"
|
377
|
+
break
|
378
|
+
end
|
379
|
+
end
|
380
|
+
self.shutdown
|
381
|
+
rescue => e
|
382
|
+
$log.warn e
|
383
|
+
end
|
384
|
+
|
385
|
+
def shutdown
|
386
|
+
@state = :closed
|
387
|
+
if @thread == Thread.current
|
388
|
+
@socket.close
|
389
|
+
@thread.kill
|
390
|
+
else
|
391
|
+
if @thread
|
392
|
+
@thread.kill
|
393
|
+
@thread.join
|
394
|
+
end
|
395
|
+
@socket.close
|
396
|
+
end
|
397
|
+
rescue => e
|
398
|
+
$log.debug "#{e.class}:#{e.message}"
|
399
|
+
end
|
400
|
+
end
|
401
|
+
end
|
402
|
+
end
|
@@ -0,0 +1,417 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require 'fluent/mixin/config_placeholders'
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
class SecureForwardOutput < ObjectBufferedOutput
|
7
|
+
DEFAULT_SECURE_CONNECT_PORT = 24284
|
8
|
+
|
9
|
+
Fluent::Plugin.register_output('secure_forward', self)
|
10
|
+
|
11
|
+
config_param :self_hostname, :string
|
12
|
+
include Fluent::Mixin::ConfigPlaceholders
|
13
|
+
|
14
|
+
config_param :shared_key, :string
|
15
|
+
|
16
|
+
# config_param :keepalive, :time, :default => 3600 # 0 means disable keepalive
|
17
|
+
|
18
|
+
config_param :send_timeout, :time, :default => 60
|
19
|
+
# config_param :hard_timeout, :time, :default => 60
|
20
|
+
# config_param :expire_dns_cache, :time, :default => 0 # 0 means disable cache
|
21
|
+
|
22
|
+
config_param :allow_self_signed_certificate, :bool, :default => true
|
23
|
+
config_param :ca_file_path, :string, :default => nil
|
24
|
+
|
25
|
+
config_param :read_length, :size, :default => 512 # 512bytes
|
26
|
+
config_param :read_interval_msec, :integer, :default => 50 # 50ms
|
27
|
+
config_param :socket_interval_msec, :integer, :default => 200 # 200ms
|
28
|
+
|
29
|
+
config_param :reconnect_interval, :time, :default => 15
|
30
|
+
|
31
|
+
attr_reader :read_interval, :socket_interval
|
32
|
+
|
33
|
+
attr_reader :nodes
|
34
|
+
# <server>
|
35
|
+
# host ipaddr/hostname
|
36
|
+
# hostlabel labelname # certification common name
|
37
|
+
# port 24284
|
38
|
+
# shared_key .... # optional shared key
|
39
|
+
# username name # if required
|
40
|
+
# password pass # if required
|
41
|
+
# </server>
|
42
|
+
|
43
|
+
def initialize
|
44
|
+
super
|
45
|
+
require 'socket'
|
46
|
+
require 'openssl'
|
47
|
+
require 'digest'
|
48
|
+
end
|
49
|
+
|
50
|
+
def configure(conf)
|
51
|
+
super
|
52
|
+
|
53
|
+
unless @allow_self_signed_certificate
|
54
|
+
raise Fluent::ConfigError, "not tested yet!"
|
55
|
+
end
|
56
|
+
|
57
|
+
@read_interval = @read_interval_msec / 1000.0
|
58
|
+
@socket_interval = @socket_interval_msec / 1000.0
|
59
|
+
|
60
|
+
# read <server> tags and set to nodes
|
61
|
+
@nodes = []
|
62
|
+
conf.elements.each do |element|
|
63
|
+
case element.name
|
64
|
+
when 'server'
|
65
|
+
unless element['host']
|
66
|
+
raise Fluent::ConfigError, "host missing in <server>"
|
67
|
+
end
|
68
|
+
node_shared_key = element['shared_key'] || @shared_key
|
69
|
+
@nodes.push Node.new(self, node_shared_key, element)
|
70
|
+
else
|
71
|
+
raise Fluent::ConfigError, "unknown config tag name #{element.name}"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
if @nodes.size > 1
|
75
|
+
raise Fluent::ConfigError, "Two or more servers are not supported yet."
|
76
|
+
end
|
77
|
+
|
78
|
+
true
|
79
|
+
end
|
80
|
+
|
81
|
+
def select_node
|
82
|
+
#TODO: roundrobin? random?
|
83
|
+
@nodes.select(&:established?).first
|
84
|
+
end
|
85
|
+
|
86
|
+
def start
|
87
|
+
super
|
88
|
+
|
89
|
+
OpenSSL::Random.seed(File.read("/dev/random", 16))
|
90
|
+
@nodes.each do |node|
|
91
|
+
node.start
|
92
|
+
end
|
93
|
+
@nodewatcher = Thread.new(&method(:node_watcher))
|
94
|
+
end
|
95
|
+
|
96
|
+
def node_watcher
|
97
|
+
loop do
|
98
|
+
sleep @reconnect_interval
|
99
|
+
$log.debug "in node health watcher"
|
100
|
+
(0...(@nodes.size)).each do |i|
|
101
|
+
$log.debug "node health watcher for #{@nodes[i].host}"
|
102
|
+
if @nodes[i].state != :established
|
103
|
+
$log.info "dead connection found: #{@nodes[i].host}, reconnecting..."
|
104
|
+
node = @nodes[i]
|
105
|
+
@nodes[i] = node.dup
|
106
|
+
@nodes[i].start
|
107
|
+
node.shutdown
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def shutdown
|
114
|
+
@nodewatcher.kill
|
115
|
+
@nodewatcher.join
|
116
|
+
@nodes.each do |node|
|
117
|
+
node.shutdown
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def write_objects(tag, es)
|
122
|
+
#TODO: check errors
|
123
|
+
node = select_node
|
124
|
+
unless node
|
125
|
+
raise "no one nodes with valid ssl session"
|
126
|
+
end
|
127
|
+
|
128
|
+
begin
|
129
|
+
send_data(node, tag, es)
|
130
|
+
rescue IOError => e
|
131
|
+
$log.warn "Failed to send messages to #{node.host}, parging."
|
132
|
+
node.shutdown
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
# MessagePack FixArray length = 2
|
137
|
+
FORWARD_HEADER = [0x92].pack('C')
|
138
|
+
|
139
|
+
# to forward messages
|
140
|
+
def send_data(node, tag, es)
|
141
|
+
ssl = node.sslsession
|
142
|
+
# beginArray(2)
|
143
|
+
ssl.write FORWARD_HEADER
|
144
|
+
|
145
|
+
# writeRaw(tag)
|
146
|
+
ssl.write tag.to_msgpack
|
147
|
+
|
148
|
+
# beginRaw(size)
|
149
|
+
sz = es.size
|
150
|
+
# # FixRaw
|
151
|
+
# ssl.write [0xa0 | sz].pack('C')
|
152
|
+
#elsif sz < 65536
|
153
|
+
# # raw 16
|
154
|
+
# ssl.write [0xda, sz].pack('Cn')
|
155
|
+
#else
|
156
|
+
# raw 32
|
157
|
+
ssl.write [0xdb, sz].pack('CN')
|
158
|
+
#end
|
159
|
+
|
160
|
+
# writeRawBody(packed_es)
|
161
|
+
es.write_to(ssl)
|
162
|
+
end
|
163
|
+
|
164
|
+
class Node # Fluent::SecureForwardOutput::Node
|
165
|
+
attr_accessor :host, :port, :hostlabel, :shared_key, :username, :password
|
166
|
+
attr_accessor :authentication, :keepalive
|
167
|
+
attr_accessor :socket, :sslsession, :unpacker, :shared_key_salt, :state
|
168
|
+
|
169
|
+
def initialize(sender, shared_key, conf)
|
170
|
+
@sender = sender
|
171
|
+
@shared_key = shared_key
|
172
|
+
|
173
|
+
@host = conf['host']
|
174
|
+
@port = (conf['port'] || DEFAULT_SECURE_CONNECT_PORT).to_i
|
175
|
+
@hostlabel = conf['hostlabel'] || conf['host']
|
176
|
+
@username = conf['username'] || ''
|
177
|
+
@password = conf['password'] || ''
|
178
|
+
|
179
|
+
@authentication = nil
|
180
|
+
@keepalive = nil
|
181
|
+
|
182
|
+
@socket = nil
|
183
|
+
@sslsession = nil
|
184
|
+
@unpacker = MessagePack::Unpacker.new
|
185
|
+
|
186
|
+
@shared_key_salt = generate_salt
|
187
|
+
@state = :helo
|
188
|
+
@thread = nil
|
189
|
+
end
|
190
|
+
|
191
|
+
def dup
|
192
|
+
Node.new(
|
193
|
+
@sender,
|
194
|
+
@shared_key,
|
195
|
+
{'host' => @host, 'port' => @port, 'hostlabel' => @hostlabel, 'username' => @username, 'password' => @password}
|
196
|
+
)
|
197
|
+
end
|
198
|
+
|
199
|
+
def start
|
200
|
+
@thread = Thread.new(&method(:connect))
|
201
|
+
end
|
202
|
+
|
203
|
+
def shutdown
|
204
|
+
$log.debug "shutting down node #{@host}"
|
205
|
+
@state = :closed
|
206
|
+
|
207
|
+
if @thread == Thread.current
|
208
|
+
@sslsession.close if @sslsession
|
209
|
+
@socket.close if @socket
|
210
|
+
@thread.kill
|
211
|
+
else
|
212
|
+
if @thread
|
213
|
+
@thread.kill
|
214
|
+
@thread.join
|
215
|
+
end
|
216
|
+
@sslsession.close if @sslsession
|
217
|
+
@socket.close if @socket
|
218
|
+
end
|
219
|
+
rescue => e
|
220
|
+
$log.debug "#{e.class}:#{e.message}"
|
221
|
+
end
|
222
|
+
|
223
|
+
def verify_result_name(code)
|
224
|
+
case code
|
225
|
+
when OpenSSL::X509::V_OK then 'V_OK'
|
226
|
+
when OpenSSL::X509::V_ERR_AKID_SKID_MISMATCH then 'V_ERR_AKID_SKID_MISMATCH'
|
227
|
+
when OpenSSL::X509::V_ERR_APPLICATION_VERIFICATION then 'V_ERR_APPLICATION_VERIFICATION'
|
228
|
+
when OpenSSL::X509::V_ERR_CERT_CHAIN_TOO_LONG then 'V_ERR_CERT_CHAIN_TOO_LONG'
|
229
|
+
when OpenSSL::X509::V_ERR_CERT_HAS_EXPIRED then 'V_ERR_CERT_HAS_EXPIRED'
|
230
|
+
when OpenSSL::X509::V_ERR_CERT_NOT_YET_VALID then 'V_ERR_CERT_NOT_YET_VALID'
|
231
|
+
when OpenSSL::X509::V_ERR_CERT_REJECTED then 'V_ERR_CERT_REJECTED'
|
232
|
+
when OpenSSL::X509::V_ERR_CERT_REVOKED then 'V_ERR_CERT_REVOKED'
|
233
|
+
when OpenSSL::X509::V_ERR_CERT_SIGNATURE_FAILURE then 'V_ERR_CERT_SIGNATURE_FAILURE'
|
234
|
+
when OpenSSL::X509::V_ERR_CERT_UNTRUSTED then 'V_ERR_CERT_UNTRUSTED'
|
235
|
+
when OpenSSL::X509::V_ERR_CRL_HAS_EXPIRED then 'V_ERR_CRL_HAS_EXPIRED'
|
236
|
+
when OpenSSL::X509::V_ERR_CRL_NOT_YET_VALID then 'V_ERR_CRL_NOT_YET_VALID'
|
237
|
+
when OpenSSL::X509::V_ERR_CRL_SIGNATURE_FAILURE then 'V_ERR_CRL_SIGNATURE_FAILURE'
|
238
|
+
when OpenSSL::X509::V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT then 'V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT'
|
239
|
+
when OpenSSL::X509::V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD then 'V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD'
|
240
|
+
when OpenSSL::X509::V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD then 'V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD'
|
241
|
+
when OpenSSL::X509::V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD then 'V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD'
|
242
|
+
when OpenSSL::X509::V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD then 'V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD'
|
243
|
+
when OpenSSL::X509::V_ERR_INVALID_CA then 'V_ERR_INVALID_CA'
|
244
|
+
when OpenSSL::X509::V_ERR_INVALID_PURPOSE then 'V_ERR_INVALID_PURPOSE'
|
245
|
+
when OpenSSL::X509::V_ERR_KEYUSAGE_NO_CERTSIGN then 'V_ERR_KEYUSAGE_NO_CERTSIGN'
|
246
|
+
when OpenSSL::X509::V_ERR_OUT_OF_MEM then 'V_ERR_OUT_OF_MEM'
|
247
|
+
when OpenSSL::X509::V_ERR_PATH_LENGTH_EXCEEDED then 'V_ERR_PATH_LENGTH_EXCEEDED'
|
248
|
+
when OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN then 'V_ERR_SELF_SIGNED_CERT_IN_CHAIN'
|
249
|
+
when OpenSSL::X509::V_ERR_SUBJECT_ISSUER_MISMATCH then 'V_ERR_SUBJECT_ISSUER_MISMATCH'
|
250
|
+
when OpenSSL::X509::V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY then 'V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY'
|
251
|
+
when OpenSSL::X509::V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE then 'V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY'
|
252
|
+
when OpenSSL::X509::V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE then 'V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE'
|
253
|
+
when OpenSSL::X509::V_ERR_UNABLE_TO_GET_CRL then 'V_ERR_UNABLE_TO_GET_CRL'
|
254
|
+
when OpenSSL::X509::V_ERR_UNABLE_TO_GET_ISSUER_CERT then 'V_ERR_UNABLE_TO_GET_ISSUER_CERT'
|
255
|
+
when OpenSSL::X509::V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY then 'V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY'
|
256
|
+
when OpenSSL::X509::V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE then 'V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE'
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
def established?
|
261
|
+
@state == :established
|
262
|
+
end
|
263
|
+
|
264
|
+
def generate_salt
|
265
|
+
OpenSSL::Random.random_bytes(16)
|
266
|
+
end
|
267
|
+
|
268
|
+
def check_helo(message)
|
269
|
+
$log.debug "checking helo"
|
270
|
+
# ['HELO', options(hash)]
|
271
|
+
unless message.size == 2 && message[0] == 'HELO'
|
272
|
+
return false
|
273
|
+
end
|
274
|
+
opts = message[1]
|
275
|
+
@authentication = opts['auth']
|
276
|
+
@keepalive = opts['keepalive']
|
277
|
+
true
|
278
|
+
end
|
279
|
+
|
280
|
+
def generate_ping
|
281
|
+
$log.debug "generating ping"
|
282
|
+
# ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + shared_key),
|
283
|
+
# username || '', sha512\_hex(auth\_salt + username + password) || '']
|
284
|
+
shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt).update(@sender.self_hostname).update(@shared_key).hexdigest
|
285
|
+
ping = ['PING', @sender.self_hostname, @shared_key_salt, shared_key_hexdigest]
|
286
|
+
if @authentication != ''
|
287
|
+
password_hexdigest = Digest::SHA512.new.update(@authentication).update(@username).update(@password).hexdigest
|
288
|
+
ping.push(@username, password_hexdigest)
|
289
|
+
else
|
290
|
+
ping.push('','')
|
291
|
+
end
|
292
|
+
ping
|
293
|
+
end
|
294
|
+
|
295
|
+
def check_pong(message)
|
296
|
+
$log.debug "checking pong"
|
297
|
+
# ['PONG', bool(authentication result), 'reason if authentication failed',
|
298
|
+
# self_hostname, sha512\_hex(salt + self_hostname + sharedkey)]
|
299
|
+
unless message.size == 5 && message[0] == 'PONG'
|
300
|
+
return false, 'invalid format for PONG message'
|
301
|
+
end
|
302
|
+
pong, auth_result, reason, hostname, shared_key_hexdigest = message
|
303
|
+
|
304
|
+
unless auth_result
|
305
|
+
return false, 'authentication failed: ' + reason
|
306
|
+
end
|
307
|
+
|
308
|
+
clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(@shared_key).hexdigest
|
309
|
+
unless shared_key_hexdigest == clientside
|
310
|
+
return false, 'shared key mismatch'
|
311
|
+
end
|
312
|
+
|
313
|
+
return true, nil
|
314
|
+
end
|
315
|
+
|
316
|
+
def send_data(data)
|
317
|
+
@sslsession.write data.to_msgpack
|
318
|
+
end
|
319
|
+
|
320
|
+
def on_read(data)
|
321
|
+
$log.debug "on_read"
|
322
|
+
if self.established?
|
323
|
+
#TODO: ACK
|
324
|
+
$log.warn "unknown packets arrived..."
|
325
|
+
return
|
326
|
+
end
|
327
|
+
|
328
|
+
case @state
|
329
|
+
when :helo
|
330
|
+
# TODO: log debug
|
331
|
+
unless check_helo(data)
|
332
|
+
$log.warn "received invalid helo message from #{@host}"
|
333
|
+
self.shutdown
|
334
|
+
return
|
335
|
+
end
|
336
|
+
send_data generate_ping()
|
337
|
+
@state = :pingpong
|
338
|
+
when :pingpong
|
339
|
+
success, reason = check_pong(data)
|
340
|
+
unless success
|
341
|
+
$log.warn "connection refused to #{@host}:" + reason
|
342
|
+
self.shutdown
|
343
|
+
return
|
344
|
+
end
|
345
|
+
$log.info "connection established to #{@host}"
|
346
|
+
@state = :established
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
350
|
+
def connect
|
351
|
+
$log.debug "starting client"
|
352
|
+
sock = TCPSocket.new(@host, @port)
|
353
|
+
|
354
|
+
opt = [1, @sender.send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; }
|
355
|
+
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
356
|
+
|
357
|
+
opt = [@sender.send_timeout.to_i, 0].pack('L!L!') # struct timeval
|
358
|
+
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
|
359
|
+
|
360
|
+
# TODO: SSLContext constructer parameter (SSL/TLS protocol version)
|
361
|
+
context = OpenSSL::SSL::SSLContext.new
|
362
|
+
context.ca_file = @cert_file_path
|
363
|
+
# TODO: context.ciphers= (SSL Shared key chiper protocols)
|
364
|
+
|
365
|
+
sslsession = OpenSSL::SSL::SSLSocket.new(sock, context)
|
366
|
+
sslsession.connect
|
367
|
+
|
368
|
+
begin
|
369
|
+
unless @sender.allow_self_signed_certificate
|
370
|
+
$log.debug sslsession.peer_cert.subject.to_s
|
371
|
+
sslsession.post_connection_check(@hostlabel)
|
372
|
+
verify = sslsession.verify_result
|
373
|
+
if verify != OpenSSL::X509::V_OK
|
374
|
+
err_name = verify_result_name(verify)
|
375
|
+
$log.warn "failed to verify certification while connecting host #{@host} as #{@hostlabel} (but not raised, why?)"
|
376
|
+
$log.warn "verify_result: #{err_name}"
|
377
|
+
raise RuntimeError, "failed to verify certification while connecting host #{@host} as #{@hostlabel}"
|
378
|
+
end
|
379
|
+
end
|
380
|
+
rescue OpenSSL::SSL::SSLError => e
|
381
|
+
$log.warn "failed to verify certification while connecting host #{@host} as #{@hostlabel}"
|
382
|
+
self.shutdown
|
383
|
+
raise
|
384
|
+
end
|
385
|
+
|
386
|
+
$log.debug "ssl sessison connected"
|
387
|
+
@socket = sock
|
388
|
+
@sslsession = sslsession
|
389
|
+
|
390
|
+
buf = ''
|
391
|
+
read_length = @sender.read_length
|
392
|
+
read_interval = @sender.read_interval
|
393
|
+
socket_interval = @sender.socket_interval
|
394
|
+
|
395
|
+
loop do
|
396
|
+
begin
|
397
|
+
while @sslsession.read_nonblock(read_length, buf)
|
398
|
+
if buf == ''
|
399
|
+
sleep read_interval
|
400
|
+
next
|
401
|
+
end
|
402
|
+
@unpacker.feed_each(buf, &method(:on_read))
|
403
|
+
buf = ''
|
404
|
+
end
|
405
|
+
rescue OpenSSL::SSL::SSLError
|
406
|
+
# to wait i/o restart
|
407
|
+
sleep socket_interval
|
408
|
+
rescue EOFError
|
409
|
+
$log.warn "disconnected from #{@host}"
|
410
|
+
break
|
411
|
+
end
|
412
|
+
end
|
413
|
+
self.shutdown
|
414
|
+
end
|
415
|
+
end
|
416
|
+
end
|
417
|
+
end
|