fluentd 0.14.4-x86-mingw32 → 0.14.5-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/ChangeLog +18 -0
- data/example/in_forward.conf +3 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_users.conf +24 -0
- data/example/out_forward.conf +13 -13
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_users.conf +65 -0
- data/example/{out_buffered_null.conf → out_null.conf} +10 -6
- data/example/secondary_file.conf +41 -0
- data/lib/fluent/agent.rb +3 -1
- data/lib/fluent/plugin/buffer.rb +5 -1
- data/lib/fluent/plugin/in_forward.rb +300 -50
- data/lib/fluent/plugin/in_tail.rb +41 -85
- data/lib/fluent/plugin/multi_output.rb +4 -0
- data/lib/fluent/plugin/out_forward.rb +326 -209
- data/lib/fluent/plugin/out_null.rb +37 -0
- data/lib/fluent/plugin/out_secondary_file.rb +128 -0
- data/lib/fluent/plugin/out_stdout.rb +38 -2
- data/lib/fluent/plugin/output.rb +13 -5
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/test/startup_shutdown.rb +33 -0
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/test_in_forward.rb +906 -441
- data/test/plugin/test_in_monitor_agent.rb +4 -0
- data/test/plugin/test_in_tail.rb +681 -663
- data/test/plugin/test_out_forward.rb +150 -208
- data/test/plugin/test_out_null.rb +85 -9
- data/test/plugin/test_out_secondary_file.rb +432 -0
- data/test/plugin/test_out_stdout.rb +143 -45
- data/test/test_root_agent.rb +42 -0
- metadata +14 -9
- data/lib/fluent/plugin/out_buffered_null.rb +0 -59
- data/lib/fluent/plugin/out_buffered_stdout.rb +0 -70
- data/test/plugin/test_out_buffered_null.rb +0 -79
- data/test/plugin/test_out_buffered_stdout.rb +0 -122
data/lib/fluent/plugin/buffer.rb
CHANGED
@@ -55,7 +55,11 @@ module Fluent
|
|
55
55
|
# if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
|
56
56
|
config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
|
57
57
|
|
58
|
-
Metadata = Struct.new(:timekey, :tag, :variables)
|
58
|
+
Metadata = Struct.new(:timekey, :tag, :variables) do
|
59
|
+
def empty?
|
60
|
+
timekey.nil? && tag.nil? && variables.nil?
|
61
|
+
end
|
62
|
+
end
|
59
63
|
|
60
64
|
# for tests
|
61
65
|
attr_accessor :stage_size, :queue_size
|
@@ -42,6 +42,8 @@ module Fluent
|
|
42
42
|
config_param :linger_timeout, :integer, default: 0
|
43
43
|
# This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
|
44
44
|
config_param :blocking_timeout, :time, default: 0.5
|
45
|
+
desc 'Connections will be disconnected right after receiving first message if this value is true.'
|
46
|
+
config_param :deny_keepalive, :bool, default: false
|
45
47
|
|
46
48
|
desc 'Log warning if received chunk size is larger than this value.'
|
47
49
|
config_param :chunk_size_warn_limit, :size, default: nil
|
@@ -52,8 +54,77 @@ module Fluent
|
|
52
54
|
desc "The field name of the client's hostname."
|
53
55
|
config_param :source_hostname_key, :string, default: nil
|
54
56
|
|
57
|
+
config_section :security, required: false, multi: false do
|
58
|
+
desc 'The hostname'
|
59
|
+
config_param :self_hostname, :string
|
60
|
+
desc 'Shared key for authentication'
|
61
|
+
config_param :shared_key, :string
|
62
|
+
desc 'If true, use user based authentication'
|
63
|
+
config_param :user_auth, :bool, default: false
|
64
|
+
desc 'Allow anonymous source. <client> sections required if disabled.'
|
65
|
+
config_param :allow_anonymous_source, :bool, default: true
|
66
|
+
|
67
|
+
### User based authentication
|
68
|
+
config_section :user, param_name: :users, required: false, multi: true do
|
69
|
+
desc 'The username for authentication'
|
70
|
+
config_param :username, :string
|
71
|
+
desc 'The password for authentication'
|
72
|
+
config_param :password, :string
|
73
|
+
end
|
74
|
+
|
75
|
+
### Client ip/network authentication & per_host shared key
|
76
|
+
config_section :client, param_name: :clients, required: false, multi: true do
|
77
|
+
desc 'The IP address or host name of the client'
|
78
|
+
config_param :host, :string, default: nil
|
79
|
+
desc 'Network address specification'
|
80
|
+
config_param :network, :string, default: nil
|
81
|
+
desc 'Shared key per client'
|
82
|
+
config_param :shared_key, :string, default: nil
|
83
|
+
desc 'Array of username.'
|
84
|
+
config_param :users, :array, default: []
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
55
88
|
def configure(conf)
|
56
89
|
super
|
90
|
+
|
91
|
+
if @security
|
92
|
+
if @security.user_auth && @security.users.empty?
|
93
|
+
raise Fluent::ConfigError, "<user> sections required if user_auth enabled"
|
94
|
+
end
|
95
|
+
if !@security.allow_anonymous_source && @security.clients.empty?
|
96
|
+
raise Fluent::ConfigError, "<client> sections required if allow_anonymous_source disabled"
|
97
|
+
end
|
98
|
+
|
99
|
+
@nodes = []
|
100
|
+
|
101
|
+
@security.clients.each do |client|
|
102
|
+
if client.host && client.network
|
103
|
+
raise Fluent::ConfigError, "both of 'host' and 'network' are specified for client"
|
104
|
+
end
|
105
|
+
if !client.host && !client.network
|
106
|
+
raise Fluent::ConfigError, "Either of 'host' and 'network' must be specified for client"
|
107
|
+
end
|
108
|
+
source = nil
|
109
|
+
if client.host
|
110
|
+
begin
|
111
|
+
source = IPSocket.getaddress(client.host)
|
112
|
+
rescue SocketError => e
|
113
|
+
raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
|
114
|
+
end
|
115
|
+
end
|
116
|
+
source_addr = begin
|
117
|
+
IPAddr.new(source || client.network)
|
118
|
+
rescue ArgumentError => e
|
119
|
+
raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
|
120
|
+
end
|
121
|
+
@nodes.push({
|
122
|
+
address: source_addr,
|
123
|
+
shared_key: (client.shared_key || @security.shared_key),
|
124
|
+
users: client.users
|
125
|
+
})
|
126
|
+
end
|
127
|
+
end
|
57
128
|
end
|
58
129
|
|
59
130
|
def start
|
@@ -100,7 +171,7 @@ module Fluent
|
|
100
171
|
def listen(client)
|
101
172
|
log.info "listening fluent socket on #{@bind}:#{@port}"
|
102
173
|
sock = client.listen_tcp(@bind, @port)
|
103
|
-
s = Coolio::TCPServer.new(sock, nil, Handler, @linger_timeout, log, method(:
|
174
|
+
s = Coolio::TCPServer.new(sock, nil, Handler, @linger_timeout, log, method(:handle_connection))
|
104
175
|
s.listen(@backlog) unless @backlog.nil?
|
105
176
|
s
|
106
177
|
end
|
@@ -124,6 +195,99 @@ module Fluent
|
|
124
195
|
|
125
196
|
private
|
126
197
|
|
198
|
+
def handle_connection(conn)
|
199
|
+
send_data = ->(serializer, data){ conn.write serializer.call(data) }
|
200
|
+
|
201
|
+
log.trace "connected fluent socket", address: conn.remote_addr, port: conn.remote_port
|
202
|
+
state = :established
|
203
|
+
nonce = nil
|
204
|
+
user_auth_salt = nil
|
205
|
+
|
206
|
+
if @security
|
207
|
+
# security enabled session MUST use MessagePack as serialization format
|
208
|
+
state = :helo
|
209
|
+
nonce = generate_salt
|
210
|
+
user_auth_salt = generate_salt
|
211
|
+
send_data.call(:to_msgpack.to_proc, generate_helo(nonce, user_auth_salt))
|
212
|
+
state = :pingpong
|
213
|
+
end
|
214
|
+
|
215
|
+
log.trace "accepted fluent socket", address: conn.remote_addr, port: conn.remote_port
|
216
|
+
|
217
|
+
read_messages(conn) do |msg, chunk_size, serializer|
|
218
|
+
case state
|
219
|
+
when :pingpong
|
220
|
+
success, reason_or_salt, shared_key = check_ping(msg, conn.remote_addr, user_auth_salt, nonce)
|
221
|
+
unless success
|
222
|
+
send_data.call(serializer, generate_pong(false, reason_or_salt, nonce, shared_key))
|
223
|
+
conn.close
|
224
|
+
next
|
225
|
+
end
|
226
|
+
send_data.call(serializer, generate_pong(true, reason_or_salt, nonce, shared_key))
|
227
|
+
|
228
|
+
log.debug "connection established", address: conn.remote_addr, port: conn.remote_port
|
229
|
+
state = :established
|
230
|
+
when :established
|
231
|
+
options = on_message(msg, chunk_size, conn.remote_addr)
|
232
|
+
if options && r = response(options)
|
233
|
+
send_data.call(serializer, r)
|
234
|
+
log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
|
235
|
+
if @deny_keepalive
|
236
|
+
conn.on_write_complete do
|
237
|
+
conn.close
|
238
|
+
end
|
239
|
+
end
|
240
|
+
else
|
241
|
+
if @deny_keepalive
|
242
|
+
conn.close
|
243
|
+
end
|
244
|
+
end
|
245
|
+
else
|
246
|
+
raise "BUG: unknown session state: #{state}"
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
def read_messages(conn, &block)
|
252
|
+
feeder = nil
|
253
|
+
serializer = nil
|
254
|
+
bytes = 0
|
255
|
+
conn.on_data do |data|
|
256
|
+
# only for first call of callback
|
257
|
+
unless feeder
|
258
|
+
first = data[0]
|
259
|
+
if first == '{' || first == '[' # json
|
260
|
+
parser = Yajl::Parser.new
|
261
|
+
parser.on_parse_complete = ->(obj){
|
262
|
+
block.call(obj, bytes, serializer)
|
263
|
+
bytes = 0
|
264
|
+
}
|
265
|
+
serializer = :to_json.to_proc
|
266
|
+
feeder = ->(d){ parser << d }
|
267
|
+
else # msgpack
|
268
|
+
parser = Fluent::Engine.msgpack_factory.unpacker
|
269
|
+
serializer = :to_msgpack.to_proc
|
270
|
+
feeder = ->(d){
|
271
|
+
parser.feed_each(d){|obj|
|
272
|
+
block.call(obj, bytes, serializer)
|
273
|
+
bytes = 0
|
274
|
+
}
|
275
|
+
}
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
279
|
+
bytes += data.bytesize
|
280
|
+
feeder.call(data)
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
def response(option)
|
285
|
+
if option && option['chunk']
|
286
|
+
return { 'ack' => option['chunk'] }
|
287
|
+
end
|
288
|
+
nil
|
289
|
+
end
|
290
|
+
|
127
291
|
# message Entry {
|
128
292
|
# 1: long time
|
129
293
|
# 2: object record
|
@@ -169,7 +333,8 @@ module Fluent
|
|
169
333
|
log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source_message(peeraddr), limit: @chunk_size_warn_limit, size: chunk_size
|
170
334
|
end
|
171
335
|
|
172
|
-
|
336
|
+
case entries
|
337
|
+
when String
|
173
338
|
# PackedForward
|
174
339
|
option = msg[2]
|
175
340
|
size = (option && option['size']) || 0
|
@@ -178,7 +343,7 @@ module Fluent
|
|
178
343
|
es = add_source_host(es, peeraddr[2]) if @source_hostname_key
|
179
344
|
router.emit_stream(tag, es)
|
180
345
|
|
181
|
-
|
346
|
+
when Array
|
182
347
|
# Forward
|
183
348
|
es = if @skip_invalid_event
|
184
349
|
check_and_skip_invalid_event(tag, entries, peeraddr)
|
@@ -246,10 +411,105 @@ module Fluent
|
|
246
411
|
"host: #{host}, addr: #{addr}, port: #{port}"
|
247
412
|
end
|
248
413
|
|
414
|
+
def select_authenticate_users(node, username)
|
415
|
+
if node.nil? || node[:users].empty?
|
416
|
+
@security.users.select{|u| u.username == username}
|
417
|
+
else
|
418
|
+
@security.users.select{|u| node[:users].include?(u.username) && u.username == username}
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
422
|
+
def generate_salt
|
423
|
+
OpenSSL::Random.random_bytes(16)
|
424
|
+
end
|
425
|
+
|
426
|
+
def generate_helo(nonce, user_auth_salt)
|
427
|
+
log.debug "generating helo"
|
428
|
+
# ['HELO', options(hash)]
|
429
|
+
['HELO', {'nonce' => nonce, 'auth' => (@security ? user_auth_salt : ''), 'keepalive' => !@deny_keepalive}]
|
430
|
+
end
|
431
|
+
|
432
|
+
##### Authentication Handshake
|
433
|
+
#
|
434
|
+
# 1. (client) connect to server
|
435
|
+
# * Socket handshake, checks certificate and its significate (in client, if using SSL)
|
436
|
+
# 2. (server)
|
437
|
+
# * check network/domain acl (if enabled)
|
438
|
+
# * disconnect when failed
|
439
|
+
# 3. (server) send HELO
|
440
|
+
# * ['HELO', options(hash)]
|
441
|
+
# * options:
|
442
|
+
# * nonce: string (required)
|
443
|
+
# * auth: string or blank_string (string: authentication required, and its salt is this value)
|
444
|
+
# 4. (client) send PING
|
445
|
+
# * ['PING', selfhostname, sharedkey_salt, sha512_hex(sharedkey_salt + selfhostname + nonce + sharedkey), username || '', sha512_hex(auth_salt + username + password) || '']
|
446
|
+
# 5. (server) check PING
|
447
|
+
# * check sharedkey
|
448
|
+
# * check username / password (if required)
|
449
|
+
# * send PONG FAILURE if failed
|
450
|
+
# * ['PONG', false, 'reason of authentication failure', '', '']
|
451
|
+
# 6. (server) send PONG
|
452
|
+
# * ['PONG', bool(authentication result), 'reason if authentication failed', selfhostname, sha512_hex(salt + selfhostname + nonce + sharedkey)]
|
453
|
+
# 7. (client) check PONG
|
454
|
+
# * check sharedkey
|
455
|
+
# * disconnect when failed
|
456
|
+
# 8. connection established
|
457
|
+
# * send data from client
|
458
|
+
|
459
|
+
def check_ping(message, remote_addr, user_auth_salt, nonce)
|
460
|
+
log.debug "checking ping"
|
461
|
+
# ['PING', self_hostname, shared_key_salt, sha512_hex(shared_key_salt + self_hostname + nonce + shared_key), username || '', sha512_hex(auth_salt + username + password) || '']
|
462
|
+
unless message.size == 6 && message[0] == 'PING'
|
463
|
+
return false, 'invalid ping message'
|
464
|
+
end
|
465
|
+
_ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
|
466
|
+
|
467
|
+
node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
|
468
|
+
if !node && !@security.allow_anonymous_source
|
469
|
+
log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
|
470
|
+
return false, "anonymous source host '#{remote_addr}' denied", nil
|
471
|
+
end
|
472
|
+
|
473
|
+
shared_key = node ? node[:shared_key] : @security.shared_key
|
474
|
+
serverside = Digest::SHA512.new.update(shared_key_salt).update(hostname).update(nonce).update(shared_key).hexdigest
|
475
|
+
if shared_key_hexdigest != serverside
|
476
|
+
log.warn "Shared key mismatch", address: remote_addr, hostname: hostname
|
477
|
+
return false, 'shared_key mismatch', nil
|
478
|
+
end
|
479
|
+
|
480
|
+
if @security.user_auth
|
481
|
+
users = select_authenticate_users(node, username)
|
482
|
+
success = false
|
483
|
+
users.each do |user|
|
484
|
+
passhash = Digest::SHA512.new.update(user_auth_salt).update(username).update(user[:password]).hexdigest
|
485
|
+
success ||= (passhash == password_digest)
|
486
|
+
end
|
487
|
+
unless success
|
488
|
+
log.warn "Authentication failed", address: remote_addr, hostname: hostname, username: username
|
489
|
+
return false, 'username/password mismatch', nil
|
490
|
+
end
|
491
|
+
end
|
492
|
+
|
493
|
+
return true, shared_key_salt, shared_key
|
494
|
+
end
|
495
|
+
|
496
|
+
def generate_pong(auth_result, reason_or_salt, nonce, shared_key)
|
497
|
+
log.debug "generating pong"
|
498
|
+
# ['PONG', bool(authentication result), 'reason if authentication failed', self_hostname, sha512_hex(salt + self_hostname + nonce + sharedkey)]
|
499
|
+
unless auth_result
|
500
|
+
return ['PONG', false, reason_or_salt, '', '']
|
501
|
+
end
|
502
|
+
|
503
|
+
shared_key_digest_hex = Digest::SHA512.new.update(reason_or_salt).update(@security.self_hostname).update(nonce).update(shared_key).hexdigest
|
504
|
+
['PONG', true, '', @security.self_hostname, shared_key_digest_hex]
|
505
|
+
end
|
506
|
+
|
249
507
|
class Handler < Coolio::Socket
|
508
|
+
attr_reader :protocol, :remote_port, :remote_addr, :remote_host
|
509
|
+
|
250
510
|
PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
|
251
511
|
|
252
|
-
def initialize(io, linger_timeout, log,
|
512
|
+
def initialize(io, linger_timeout, log, on_connect_callback)
|
253
513
|
super(io)
|
254
514
|
|
255
515
|
@peeraddr = nil
|
@@ -259,8 +519,21 @@ module Fluent
|
|
259
519
|
io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
260
520
|
end
|
261
521
|
|
522
|
+
### TODO: disabling name rev resolv
|
523
|
+
proto, port, host, addr = ( io.peeraddr rescue PEERADDR_FAILED )
|
524
|
+
if addr == '?'
|
525
|
+
port, addr = *Socket.unpack_sockaddr_in(io.getpeername) rescue nil
|
526
|
+
end
|
527
|
+
@protocol = proto
|
528
|
+
@remote_port = port
|
529
|
+
@remote_addr = addr
|
530
|
+
@remote_host = host
|
531
|
+
@writing = false
|
532
|
+
@closing = false
|
533
|
+
@mutex = Mutex.new
|
534
|
+
|
262
535
|
@chunk_counter = 0
|
263
|
-
@
|
536
|
+
@on_connect_callback = on_connect_callback
|
264
537
|
@log = log
|
265
538
|
@log.trace {
|
266
539
|
begin
|
@@ -269,69 +542,46 @@ module Fluent
|
|
269
542
|
remote_port = nil
|
270
543
|
remote_addr = nil
|
271
544
|
end
|
272
|
-
"accepted fluent socket
|
545
|
+
[ "accepted fluent socket", {address: remote_addr, port: remote_port, instance: self.object_id} ]
|
273
546
|
}
|
274
547
|
end
|
275
548
|
|
276
549
|
def on_connect
|
550
|
+
@on_connect_callback.call(self)
|
277
551
|
end
|
278
552
|
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
m = method(:on_read_json)
|
283
|
-
@serializer = :to_json.to_proc
|
284
|
-
@y = Yajl::Parser.new
|
285
|
-
@y.on_parse_complete = lambda { |obj|
|
286
|
-
option = @on_message.call(obj, @chunk_counter, @peeraddr)
|
287
|
-
respond option
|
288
|
-
@chunk_counter = 0
|
289
|
-
}
|
290
|
-
else
|
291
|
-
m = method(:on_read_msgpack)
|
292
|
-
@serializer = :to_msgpack.to_proc
|
293
|
-
@u = Fluent::Engine.msgpack_factory.unpacker
|
294
|
-
end
|
295
|
-
|
296
|
-
(class << self; self; end).module_eval do
|
297
|
-
define_method(:on_read, m)
|
298
|
-
end
|
299
|
-
m.call(data)
|
553
|
+
# API to register callback for data arrival
|
554
|
+
def on_data(&callback)
|
555
|
+
@on_read_callback = callback
|
300
556
|
end
|
301
557
|
|
302
|
-
def
|
303
|
-
@
|
304
|
-
@y << data
|
558
|
+
def on_read(data)
|
559
|
+
@on_read_callback.call(data)
|
305
560
|
rescue => e
|
306
|
-
@log.error "
|
561
|
+
@log.error "unexpected error on reading data from client", address: @remote_addr, error: e
|
307
562
|
@log.error_backtrace
|
308
563
|
close
|
309
564
|
end
|
310
565
|
|
311
|
-
def
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
566
|
+
def on_write_complete
|
567
|
+
closing = @mutex.synchronize {
|
568
|
+
@writing = false
|
569
|
+
@closing
|
570
|
+
}
|
571
|
+
if closing
|
572
|
+
close
|
317
573
|
end
|
318
|
-
rescue => e
|
319
|
-
@log.error "forward error", error: e, error_class: e.class
|
320
|
-
@log.error_backtrace
|
321
|
-
close
|
322
574
|
end
|
323
575
|
|
324
|
-
def
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
576
|
+
def close
|
577
|
+
writing = @mutex.synchronize {
|
578
|
+
@closing = true
|
579
|
+
@writing
|
580
|
+
}
|
581
|
+
unless writing
|
582
|
+
super
|
329
583
|
end
|
330
584
|
end
|
331
|
-
|
332
|
-
def on_close
|
333
|
-
@log.trace { "closed socket" }
|
334
|
-
end
|
335
585
|
end
|
336
586
|
|
337
587
|
class HeartbeatRequestHandler < Coolio::IO
|
@@ -16,10 +16,9 @@
|
|
16
16
|
|
17
17
|
require 'cool.io'
|
18
18
|
|
19
|
-
require 'fluent/input'
|
19
|
+
require 'fluent/plugin/input'
|
20
20
|
require 'fluent/config/error'
|
21
21
|
require 'fluent/event'
|
22
|
-
require 'fluent/system_config'
|
23
22
|
require 'fluent/plugin/buffer'
|
24
23
|
|
25
24
|
if Fluent.windows?
|
@@ -28,11 +27,11 @@ else
|
|
28
27
|
Fluent::FileWrapper = File
|
29
28
|
end
|
30
29
|
|
31
|
-
module Fluent
|
32
|
-
class TailInput < Input
|
33
|
-
|
30
|
+
module Fluent::Plugin
|
31
|
+
class TailInput < Fluent::Plugin::Input
|
32
|
+
Fluent::Plugin.register_input('tail', self)
|
34
33
|
|
35
|
-
|
34
|
+
helpers :timer, :event_loop, :parser, :compat_parameters
|
36
35
|
|
37
36
|
FILE_PERMISSION = 0644
|
38
37
|
|
@@ -77,11 +76,24 @@ module Fluent
|
|
77
76
|
attr_reader :paths
|
78
77
|
|
79
78
|
def configure(conf)
|
79
|
+
compat_parameters_convert(conf, :parser)
|
80
|
+
parser_config = conf.elements('parse').first
|
81
|
+
unless parser_config
|
82
|
+
raise Fluent::ConfigError, "<parse> section is required."
|
83
|
+
end
|
84
|
+
unless parser_config["@type"]
|
85
|
+
raise Fluent::ConfigError, "parse/@type is required."
|
86
|
+
end
|
87
|
+
|
88
|
+
(1..Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM).each do |n|
|
89
|
+
parser_config["format#{n}"] = conf["format#{n}"] if conf["format#{n}"]
|
90
|
+
end
|
91
|
+
|
80
92
|
super
|
81
93
|
|
82
94
|
@paths = @path.split(',').map {|path| path.strip }
|
83
95
|
if @paths.empty?
|
84
|
-
raise ConfigError, "tail: 'path' parameter is required on tail input"
|
96
|
+
raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
|
85
97
|
end
|
86
98
|
|
87
99
|
unless @pos_file
|
@@ -89,22 +101,17 @@ module Fluent
|
|
89
101
|
$log.warn "this parameter is highly recommended to save the position to resume tailing."
|
90
102
|
end
|
91
103
|
|
92
|
-
configure_parser(conf)
|
93
104
|
configure_tag
|
94
105
|
configure_encoding
|
95
106
|
|
96
|
-
@multiline_mode =
|
107
|
+
@multiline_mode = parser_config["@type"] =~ /multiline/
|
97
108
|
@receive_handler = if @multiline_mode
|
98
109
|
method(:parse_multilines)
|
99
110
|
else
|
100
111
|
method(:parse_singleline)
|
101
112
|
end
|
102
113
|
@file_perm = system_config.file_permission || FILE_PERMISSION
|
103
|
-
|
104
|
-
|
105
|
-
def configure_parser(conf)
|
106
|
-
@parser = Plugin.new_parser(conf['format'])
|
107
|
-
@parser.configure(conf)
|
114
|
+
@parser = parser_create(conf: parser_config)
|
108
115
|
end
|
109
116
|
|
110
117
|
def configure_tag
|
@@ -120,7 +127,7 @@ module Fluent
|
|
120
127
|
def configure_encoding
|
121
128
|
unless @encoding
|
122
129
|
if @from_encoding
|
123
|
-
raise ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
|
130
|
+
raise Fluent::ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
|
124
131
|
end
|
125
132
|
end
|
126
133
|
|
@@ -132,7 +139,7 @@ module Fluent
|
|
132
139
|
begin
|
133
140
|
Encoding.find(encoding_name) if encoding_name
|
134
141
|
rescue ArgumentError => e
|
135
|
-
raise ConfigError, e.message
|
142
|
+
raise Fluent::ConfigError, e.message
|
136
143
|
end
|
137
144
|
end
|
138
145
|
|
@@ -145,20 +152,12 @@ module Fluent
|
|
145
152
|
@pf = PositionFile.parse(@pf_file)
|
146
153
|
end
|
147
154
|
|
148
|
-
@loop = Coolio::Loop.new
|
149
155
|
refresh_watchers
|
150
|
-
|
151
|
-
@refresh_trigger = TailWatcher::TimerWatcher.new(@refresh_interval, true, log, &method(:refresh_watchers))
|
152
|
-
@refresh_trigger.attach(@loop)
|
153
|
-
@thread = Thread.new(&method(:run))
|
156
|
+
timer_execute(:in_tail_refresh_watchers, @refresh_interval, &method(:refresh_watchers))
|
154
157
|
end
|
155
158
|
|
156
159
|
def shutdown
|
157
|
-
@refresh_trigger.detach if @refresh_trigger && @refresh_trigger.attached?
|
158
|
-
|
159
160
|
stop_watchers(@tails.keys, true)
|
160
|
-
@loop.stop rescue nil # when all watchers are detached, `stop` raises RuntimeError. We can ignore this exception.
|
161
|
-
@thread.join
|
162
161
|
@pf_file.close if @pf_file
|
163
162
|
|
164
163
|
super
|
@@ -206,8 +205,11 @@ module Fluent
|
|
206
205
|
|
207
206
|
def setup_watcher(path, pe)
|
208
207
|
line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
|
209
|
-
tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,
|
210
|
-
tw.attach
|
208
|
+
tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, &method(:receive_lines))
|
209
|
+
tw.attach do |watcher|
|
210
|
+
timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
|
211
|
+
event_loop_attach(watcher.stat_trigger)
|
212
|
+
end
|
211
213
|
tw
|
212
214
|
end
|
213
215
|
|
@@ -218,7 +220,7 @@ module Fluent
|
|
218
220
|
pe = @pf[path]
|
219
221
|
if @read_from_head && pe.read_inode.zero?
|
220
222
|
begin
|
221
|
-
pe.update(FileWrapper.stat(path).ino, 0)
|
223
|
+
pe.update(Fluent::FileWrapper.stat(path).ino, 0)
|
222
224
|
rescue Errno::ENOENT
|
223
225
|
$log.warn "#{path} not found. Continuing without tailing it."
|
224
226
|
end
|
@@ -263,8 +265,9 @@ module Fluent
|
|
263
265
|
end
|
264
266
|
|
265
267
|
def close_watcher_after_rotate_wait(tw)
|
266
|
-
|
267
|
-
|
268
|
+
timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
|
269
|
+
close_watcher(tw)
|
270
|
+
end
|
268
271
|
end
|
269
272
|
|
270
273
|
def flush_buffer(tw)
|
@@ -293,13 +296,6 @@ module Fluent
|
|
293
296
|
end
|
294
297
|
end
|
295
298
|
|
296
|
-
def run
|
297
|
-
@loop.run
|
298
|
-
rescue
|
299
|
-
log.error "unexpected error", error: $!.to_s
|
300
|
-
log.error_backtrace
|
301
|
-
end
|
302
|
-
|
303
299
|
# @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
|
304
300
|
def receive_lines(lines, tail_watcher)
|
305
301
|
es = @receive_handler.call(lines, tail_watcher)
|
@@ -347,7 +343,7 @@ module Fluent
|
|
347
343
|
end
|
348
344
|
|
349
345
|
def parse_singleline(lines, tail_watcher)
|
350
|
-
es = MultiEventStream.new
|
346
|
+
es = Fluent::MultiEventStream.new
|
351
347
|
lines.each { |line|
|
352
348
|
convert_line_to_event(line, es, tail_watcher)
|
353
349
|
}
|
@@ -356,7 +352,7 @@ module Fluent
|
|
356
352
|
|
357
353
|
def parse_multilines(lines, tail_watcher)
|
358
354
|
lb = tail_watcher.line_buffer
|
359
|
-
es = MultiEventStream.new
|
355
|
+
es = Fluent::MultiEventStream.new
|
360
356
|
if @parser.has_firstline?
|
361
357
|
tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
|
362
358
|
lines.each { |line|
|
@@ -400,8 +396,6 @@ module Fluent
|
|
400
396
|
@receive_lines = receive_lines
|
401
397
|
@update_watcher = update_watcher
|
402
398
|
|
403
|
-
@timer_trigger = TimerWatcher.new(1, true, log, &method(:on_notify)) if @enable_watch_timer
|
404
|
-
|
405
399
|
@stat_trigger = StatWatcher.new(path, log, &method(:on_notify))
|
406
400
|
|
407
401
|
@rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
|
@@ -412,6 +406,8 @@ module Fluent
|
|
412
406
|
end
|
413
407
|
|
414
408
|
attr_reader :path
|
409
|
+
attr_reader :stat_trigger, :enable_watch_timer
|
410
|
+
attr_accessor :timer_trigger
|
415
411
|
attr_accessor :line_buffer, :line_buffer_timer_flusher
|
416
412
|
attr_accessor :unwatched # This is used for removing position entry from PositionFile
|
417
413
|
|
@@ -423,14 +419,12 @@ module Fluent
|
|
423
419
|
@receive_lines.call(lines, self)
|
424
420
|
end
|
425
421
|
|
426
|
-
def attach
|
427
|
-
|
428
|
-
@stat_trigger.attach(loop)
|
422
|
+
def attach
|
423
|
+
yield self
|
429
424
|
on_notify
|
430
425
|
end
|
431
426
|
|
432
427
|
def detach
|
433
|
-
@timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
|
434
428
|
@stat_trigger.detach if @stat_trigger.attached?
|
435
429
|
end
|
436
430
|
|
@@ -523,22 +517,6 @@ module Fluent
|
|
523
517
|
pe # This pe will be updated in on_rotate after TailWatcher is initialized
|
524
518
|
end
|
525
519
|
|
526
|
-
class TimerWatcher < Coolio::TimerWatcher
|
527
|
-
def initialize(interval, repeat, log, &callback)
|
528
|
-
@callback = callback
|
529
|
-
@log = log
|
530
|
-
super(interval, repeat)
|
531
|
-
end
|
532
|
-
|
533
|
-
def on_timer
|
534
|
-
@callback.call
|
535
|
-
rescue
|
536
|
-
# TODO log?
|
537
|
-
@log.error $!.to_s
|
538
|
-
@log.error_backtrace
|
539
|
-
end
|
540
|
-
end
|
541
|
-
|
542
520
|
class StatWatcher < Coolio::StatWatcher
|
543
521
|
def initialize(path, log, &callback)
|
544
522
|
@callback = callback
|
@@ -555,24 +533,6 @@ module Fluent
|
|
555
533
|
end
|
556
534
|
end
|
557
535
|
|
558
|
-
class Closer < Coolio::TimerWatcher
|
559
|
-
def initialize(interval, tw, log, &callback)
|
560
|
-
@callback = callback
|
561
|
-
@tw = tw
|
562
|
-
@log = log
|
563
|
-
super(interval, false)
|
564
|
-
end
|
565
|
-
|
566
|
-
def on_timer
|
567
|
-
@callback.call(@tw)
|
568
|
-
rescue => e
|
569
|
-
@log.error e.to_s
|
570
|
-
@log.error_backtrace(e.backtrace)
|
571
|
-
ensure
|
572
|
-
detach
|
573
|
-
end
|
574
|
-
end
|
575
|
-
|
576
536
|
class IOHandler
|
577
537
|
def initialize(io, pe, log, read_lines_limit, first = true, &receive_lines)
|
578
538
|
@log = log
|
@@ -660,7 +620,7 @@ module Fluent
|
|
660
620
|
|
661
621
|
def on_notify
|
662
622
|
begin
|
663
|
-
stat = FileWrapper.stat(@path)
|
623
|
+
stat = Fluent::FileWrapper.stat(@path)
|
664
624
|
inode = stat.ino
|
665
625
|
fsize = stat.size
|
666
626
|
rescue Errno::ENOENT
|
@@ -673,7 +633,7 @@ module Fluent
|
|
673
633
|
if @inode != inode || fsize < @fsize
|
674
634
|
# rotated or truncated
|
675
635
|
begin
|
676
|
-
io = FileWrapper.open(@path)
|
636
|
+
io = Fluent::FileWrapper.open(@path)
|
677
637
|
rescue Errno::ENOENT
|
678
638
|
end
|
679
639
|
@on_rotate.call(io)
|
@@ -688,7 +648,6 @@ module Fluent
|
|
688
648
|
end
|
689
649
|
end
|
690
650
|
|
691
|
-
|
692
651
|
class LineBufferTimerFlusher
|
693
652
|
def initialize(log, flush_interval, &flush_method)
|
694
653
|
@log = log
|
@@ -713,7 +672,6 @@ module Fluent
|
|
713
672
|
end
|
714
673
|
end
|
715
674
|
|
716
|
-
|
717
675
|
class PositionFile
|
718
676
|
UNWATCHED_POSITION = 0xffffffffffffffff
|
719
677
|
|
@@ -833,6 +791,4 @@ module Fluent
|
|
833
791
|
end
|
834
792
|
end
|
835
793
|
end
|
836
|
-
|
837
|
-
NewTailInput = TailInput # for backward compatibility
|
838
794
|
end
|