fluentd 0.14.9 → 0.14.10
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +2 -0
- data/ChangeLog +44 -0
- data/appveyor.yml +1 -0
- data/code-of-conduct.md +3 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/command/cat.rb +11 -3
- data/lib/fluent/compat/output.rb +6 -3
- data/lib/fluent/compat/parser.rb +2 -0
- data/lib/fluent/config/section.rb +1 -1
- data/lib/fluent/env.rb +1 -1
- data/lib/fluent/plugin/filter_record_transformer.rb +12 -30
- data/lib/fluent/plugin/in_forward.rb +50 -169
- data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
- data/lib/fluent/plugin/in_syslog.rb +13 -7
- data/lib/fluent/plugin/in_tail.rb +29 -14
- data/lib/fluent/plugin/in_tcp.rb +54 -14
- data/lib/fluent/plugin/in_udp.rb +49 -13
- data/lib/fluent/plugin/out_file.rb +30 -14
- data/lib/fluent/plugin/out_forward.rb +199 -173
- data/lib/fluent/plugin/output.rb +71 -46
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin_helper.rb +2 -0
- data/lib/fluent/plugin_helper/event_loop.rb +24 -6
- data/lib/fluent/plugin_helper/inject.rb +12 -1
- data/lib/fluent/plugin_helper/server.rb +494 -0
- data/lib/fluent/plugin_helper/socket.rb +101 -0
- data/lib/fluent/plugin_helper/socket_option.rb +84 -0
- data/lib/fluent/plugin_helper/timer.rb +1 -0
- data/lib/fluent/test/driver/base.rb +45 -13
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +1 -1
- data/test/compat/test_parser.rb +10 -0
- data/test/config/test_configurable.rb +20 -0
- data/test/helper.rb +36 -1
- data/test/plugin/test_filter_record_transformer.rb +31 -103
- data/test/plugin/test_in_forward.rb +13 -75
- data/test/plugin/test_in_monitor_agent.rb +65 -35
- data/test/plugin/test_in_syslog.rb +39 -3
- data/test/plugin/test_in_tcp.rb +78 -62
- data/test/plugin/test_in_udp.rb +101 -80
- data/test/plugin/test_out_file.rb +17 -0
- data/test/plugin/test_out_forward.rb +155 -125
- data/test/plugin/test_output_as_buffered.rb +4 -2
- data/test/plugin_helper/test_inject.rb +21 -0
- data/test/plugin_helper/test_server.rb +905 -0
- data/test/test_event_time.rb +3 -1
- data/test/test_output.rb +30 -1
- data/test/test_test_drivers.rb +5 -2
- metadata +19 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3747d9d31647a7914fdf5ad62cf2a09198ff50aa
|
4
|
+
data.tar.gz: 42f7bf5f5eea0a2b0a0a4e0c99428c49659a8e60
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ef1846bbd44606aaee285205c3019279c53edc6ff5a4e70fdf8cf1af1f8460684ba9581b52127a4a4598b23046b2d9d0dcf764a0e883f0556cd3208ad2431c81
|
7
|
+
data.tar.gz: a4f6242a28705c9bec65e15a86d2b9d781394bae1f2fddec09d7f8d16fcf3a8f75c03df7f3816623067e2557991d19c3ae4f7a870e1d6bc8990a7fb4c928141d
|
data/.travis.yml
CHANGED
data/ChangeLog
CHANGED
@@ -1,5 +1,49 @@
|
|
1
1
|
# v0.14
|
2
2
|
|
3
|
+
## Release v0.14.10 - 2016/12/14
|
4
|
+
|
5
|
+
### New features / Enhancement
|
6
|
+
|
7
|
+
* Add socket/server plugin helper to write TCP/UDP clients/servers as Fluentd plugin
|
8
|
+
https://github.com/fluent/fluentd/pull/1312
|
9
|
+
https://github.com/fluent/fluentd/pull/1350
|
10
|
+
https://github.com/fluent/fluentd/pull/1356
|
11
|
+
https://github.com/fluent/fluentd/pull/1362
|
12
|
+
* Fix to raise errors when injected hostname is also specified as chunk key
|
13
|
+
https://github.com/fluent/fluentd/pull/1357
|
14
|
+
* in_tail: Optimize to read lines from file
|
15
|
+
https://github.com/fluent/fluentd/pull/1325
|
16
|
+
* in_monitor_agent: Add new parameter "include_config"(default: true)
|
17
|
+
https://github.com/fluent/fluentd/pull/1317
|
18
|
+
* in_syslog: Add "priority_key" and "facility_key" options
|
19
|
+
https://github.com/fluent/fluentd/pull/1351
|
20
|
+
* filter_record_transformer: Remove obsoleted syntax like "${message}" and not to dump records in logs
|
21
|
+
https://github.com/fluent/fluentd/pull/1328
|
22
|
+
* Add an option "--time-as-integer" to fluent-cat command to send events from v0.14 fluent-cat to v0.12 fluentd
|
23
|
+
https://github.com/fluent/fluentd/pull/1349
|
24
|
+
|
25
|
+
### Bug fixes
|
26
|
+
|
27
|
+
* Specify correct Oj options for newer versions (Oj 2.18.0 or later)
|
28
|
+
https://github.com/fluent/fluentd/pull/1331
|
29
|
+
* TimeSlice output plugins (in v0.12 style) raise errors when "utc" parameter is specified
|
30
|
+
https://github.com/fluent/fluentd/pull/1319
|
31
|
+
* Parser plugins cannot use options for regular expressions
|
32
|
+
https://github.com/fluent/fluentd/pull/1326/files
|
33
|
+
* Fix bugs not to raise errors to use logger in v0.12 plugins
|
34
|
+
https://github.com/fluent/fluentd/pull/1344
|
35
|
+
https://github.com/fluent/fluentd/pull/1332
|
36
|
+
* Fix bug about shutting down Fluentd in Windows
|
37
|
+
https://github.com/fluent/fluentd/pull/1367
|
38
|
+
* in_tail: Close files explicitly in tests
|
39
|
+
https://github.com/fluent/fluentd/pull/1327
|
40
|
+
* in_forward: Fix bug not to convert buffer configurations into v0.14 parameters
|
41
|
+
https://github.com/fluent/fluentd/pull/1337
|
42
|
+
* out_forward: Fix bug to raise error when "expire_dns_cache" is specified
|
43
|
+
https://github.com/fluent/fluentd/pull/1346
|
44
|
+
* out_file: Fix bug to raise error about buffer chunking when it's configured as secondary
|
45
|
+
https://github.com/fluent/fluentd/pull/1338
|
46
|
+
|
3
47
|
## Release v0.14.9 - 2016/11/15
|
4
48
|
|
5
49
|
### New features / Enhancement
|
data/appveyor.yml
CHANGED
data/code-of-conduct.md
ADDED
data/fluentd.gemspec
CHANGED
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
|
22
22
|
gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
|
23
23
|
gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
|
24
24
|
gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
|
25
|
-
gem.add_runtime_dependency("serverengine", ["
|
25
|
+
gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
|
26
26
|
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
|
27
27
|
gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
|
28
28
|
gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
|
data/lib/fluent/command/cat.rb
CHANGED
@@ -31,6 +31,7 @@ socket_path = Fluent::DEFAULT_SOCKET_PATH
|
|
31
31
|
config_path = Fluent::DEFAULT_CONFIG_PATH
|
32
32
|
format = 'json'
|
33
33
|
message_key = 'message'
|
34
|
+
time_as_integer = false
|
34
35
|
|
35
36
|
op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
|
36
37
|
port = i
|
@@ -68,6 +69,10 @@ op.on('--message-key KEY', "key field for none format (default: #{message_key})"
|
|
68
69
|
message_key = s
|
69
70
|
}
|
70
71
|
|
72
|
+
op.on('--time-as-integer', "Send time as integer for v0.12 or earlier", TrueClass) { |b|
|
73
|
+
time_as_integer = true
|
74
|
+
}
|
75
|
+
|
71
76
|
(class << self; self; end).module_eval do
|
72
77
|
define_method(:usage) do |msg|
|
73
78
|
puts op.to_s
|
@@ -123,7 +128,7 @@ class Writer
|
|
123
128
|
end
|
124
129
|
end
|
125
130
|
|
126
|
-
def initialize(tag, connector)
|
131
|
+
def initialize(tag, connector, time_as_integer: false)
|
127
132
|
@tag = tag
|
128
133
|
@connector = connector
|
129
134
|
@socket = false
|
@@ -136,6 +141,7 @@ class Writer
|
|
136
141
|
@pending_limit = 1024 # TODO
|
137
142
|
@retry_wait = 1
|
138
143
|
@retry_limit = 5 # TODO
|
144
|
+
@time_as_integer = time_as_integer
|
139
145
|
|
140
146
|
super()
|
141
147
|
end
|
@@ -145,7 +151,9 @@ class Writer
|
|
145
151
|
raise ArgumentError, "Input must be a map (got #{record.class})"
|
146
152
|
end
|
147
153
|
|
148
|
-
|
154
|
+
time = Fluent::EventTime.now
|
155
|
+
time = time.to_i if @time_as_integer
|
156
|
+
entry = [time, record]
|
149
157
|
synchronize {
|
150
158
|
unless write_impl([entry])
|
151
159
|
# write failed
|
@@ -275,7 +283,7 @@ else
|
|
275
283
|
}
|
276
284
|
end
|
277
285
|
|
278
|
-
w = Writer.new(tag, connector)
|
286
|
+
w = Writer.new(tag, connector, time_as_integer: time_as_integer)
|
279
287
|
w.start
|
280
288
|
|
281
289
|
case format
|
data/lib/fluent/compat/output.rb
CHANGED
@@ -81,7 +81,7 @@ module Fluent
|
|
81
81
|
end
|
82
82
|
|
83
83
|
def assume_timekey!
|
84
|
-
@_formatter = Fluent::
|
84
|
+
@_formatter = Fluent::TimeFormatter.new(@_time_slice_format, nil, @_timezone)
|
85
85
|
|
86
86
|
return if self.metadata.timekey
|
87
87
|
if self.respond_to?(:path) && self.path =~ /\.(\d+)\.(?:b|q)(?:[a-z0-9]+)/
|
@@ -619,13 +619,16 @@ module Fluent
|
|
619
619
|
if conf['timezone']
|
620
620
|
Fluent::Timezone.validate!(conf['timezone'])
|
621
621
|
elsif conf['utc']
|
622
|
-
|
622
|
+
# v0.12 assumes UTC without any configuration
|
623
|
+
# 'localtime=false && no timezone key' means UTC
|
623
624
|
conf['localtime'] = "false"
|
625
|
+
conf.delete('utc')
|
624
626
|
elsif conf['localtime']
|
625
627
|
conf['timezone'] = Time.now.strftime('%z')
|
626
628
|
conf['localtime'] = "true"
|
627
629
|
else
|
628
|
-
|
630
|
+
# v0.12 assumes UTC without any configuration
|
631
|
+
# 'localtime=false && no timezone key' means UTC
|
629
632
|
conf['localtime'] = "false"
|
630
633
|
end
|
631
634
|
|
data/lib/fluent/compat/parser.rb
CHANGED
@@ -168,6 +168,8 @@ module Fluent
|
|
168
168
|
return if @manually_configured # not to run twice
|
169
169
|
|
170
170
|
conf['expression'] ||= @stored_regexp.source
|
171
|
+
conf['ignorecase'] ||= @stored_regexp.options & Regexp::IGNORECASE != 0
|
172
|
+
conf['multiline'] ||= @stored_regexp.options & Regexp::MULTILINE != 0
|
171
173
|
convert_type_converter_parameters!(conf)
|
172
174
|
|
173
175
|
super
|
@@ -147,7 +147,7 @@ module Fluent
|
|
147
147
|
# because they are expected to be removed entirely sometime in the future.
|
148
148
|
# Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
|
149
149
|
if opts[:deprecated]
|
150
|
-
logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}"
|
150
|
+
logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}" if logger
|
151
151
|
end
|
152
152
|
if opts[:obsoleted]
|
153
153
|
logger.error "config error in:\n#{conf}" if logger
|
data/lib/fluent/env.rb
CHANGED
@@ -18,7 +18,7 @@ module Fluent
|
|
18
18
|
DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
|
19
19
|
DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
|
20
20
|
DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
|
21
|
-
DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat}
|
21
|
+
DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
|
22
22
|
IS_WINDOWS = /mswin|mingw/ === RUBY_PLATFORM
|
23
23
|
private_constant :IS_WINDOWS
|
24
24
|
|
@@ -92,23 +92,22 @@ module Fluent::Plugin
|
|
92
92
|
'tag_suffix' => tag_suffix,
|
93
93
|
'hostname' => @hostname,
|
94
94
|
}
|
95
|
-
last_record = nil
|
96
95
|
es.each do |time, record|
|
97
|
-
|
98
|
-
|
99
|
-
|
96
|
+
begin
|
97
|
+
placeholder_values['time'] = @placeholder_expander.time_value(time)
|
98
|
+
placeholder_values['record'] = record
|
100
99
|
|
101
|
-
|
102
|
-
|
103
|
-
|
100
|
+
new_record = reform(record, placeholder_values)
|
101
|
+
if @renew_time_key && new_record.has_key?(@renew_time_key)
|
102
|
+
time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
|
103
|
+
end
|
104
|
+
new_es.add(time, new_record)
|
105
|
+
rescue => e
|
106
|
+
router.emit_error_event(tag, time, record, e)
|
107
|
+
log.debug { "map:#{@map} record:#{record} placeholder_values:#{placeholder_values}" }
|
104
108
|
end
|
105
|
-
new_es.add(time, new_record)
|
106
109
|
end
|
107
110
|
new_es
|
108
|
-
rescue => e
|
109
|
-
log.warn "failed to reform records", error: e
|
110
|
-
log.warn_backtrace
|
111
|
-
log.debug "map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
|
112
111
|
end
|
113
112
|
|
114
113
|
private
|
@@ -203,9 +202,6 @@ module Fluent::Plugin
|
|
203
202
|
end
|
204
203
|
elsif value.kind_of?(Hash) # record, etc
|
205
204
|
value.each do |k, v|
|
206
|
-
unless placeholder_values.has_key?(k) # prevent overwriting reserved keys such as tag
|
207
|
-
placeholders.store("${#{k}}", v) # foo
|
208
|
-
end
|
209
205
|
placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
|
210
206
|
end
|
211
207
|
else # string, interger, float, and others?
|
@@ -308,28 +304,14 @@ module Fluent::Plugin
|
|
308
304
|
placeholders['hostname'],
|
309
305
|
)
|
310
306
|
rescue => e
|
311
|
-
|
312
|
-
log.warn_backtrace
|
313
|
-
nil
|
307
|
+
raise "failed to expand `#{str}` : error = #{e}"
|
314
308
|
end
|
315
309
|
|
316
310
|
class CleanroomExpander
|
317
311
|
def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
|
318
|
-
Thread.current[:record_transformer_record] = record # for old version compatibility
|
319
312
|
instance_eval(__str_to_eval__)
|
320
313
|
end
|
321
314
|
|
322
|
-
# for old version compatibility
|
323
|
-
def method_missing(name)
|
324
|
-
key = name.to_s
|
325
|
-
record = Thread.current[:record_transformer_record]
|
326
|
-
if record.has_key?(key)
|
327
|
-
record[key]
|
328
|
-
else
|
329
|
-
raise NameError, "undefined local variable or method `#{key}'"
|
330
|
-
end
|
331
|
-
end
|
332
|
-
|
333
315
|
(Object.instance_methods).each do |m|
|
334
316
|
undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
|
335
317
|
end
|
@@ -20,10 +20,6 @@ require 'fluent/msgpack_factory'
|
|
20
20
|
require 'yajl'
|
21
21
|
require 'digest'
|
22
22
|
|
23
|
-
require 'fluent/plugin/socket_util'
|
24
|
-
require 'fcntl'
|
25
|
-
require 'cool.io'
|
26
|
-
|
27
23
|
module Fluent::Plugin
|
28
24
|
class ForwardInput < Input
|
29
25
|
Fluent::Plugin.register_input('forward', self)
|
@@ -31,7 +27,7 @@ module Fluent::Plugin
|
|
31
27
|
# See the wiki page below for protocol specification
|
32
28
|
# https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1
|
33
29
|
|
34
|
-
helpers :
|
30
|
+
helpers :server
|
35
31
|
|
36
32
|
LISTEN_PORT = 24224
|
37
33
|
|
@@ -39,12 +35,15 @@ module Fluent::Plugin
|
|
39
35
|
config_param :port, :integer, default: LISTEN_PORT
|
40
36
|
desc 'The bind address to listen to.'
|
41
37
|
config_param :bind, :string, default: '0.0.0.0'
|
38
|
+
|
42
39
|
config_param :backlog, :integer, default: nil
|
43
40
|
# SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
|
44
41
|
desc 'The timeout time used to set linger option.'
|
45
42
|
config_param :linger_timeout, :integer, default: 0
|
46
43
|
# This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
|
47
44
|
config_param :blocking_timeout, :time, default: 0.5
|
45
|
+
desc 'Try to resolve hostname from IP addresses or not.'
|
46
|
+
config_param :resolve_hostname, :bool, default: nil
|
48
47
|
desc 'Connections will be disconnected right after receiving first message if this value is true.'
|
49
48
|
config_param :deny_keepalive, :bool, default: false
|
50
49
|
|
@@ -91,6 +90,15 @@ module Fluent::Plugin
|
|
91
90
|
def configure(conf)
|
92
91
|
super
|
93
92
|
|
93
|
+
if @source_hostname_key
|
94
|
+
# TODO: add test
|
95
|
+
if @resolve_hostname.nil?
|
96
|
+
@resolve_hostname = true
|
97
|
+
elsif !@resolve_hostname # user specifies "false" in config
|
98
|
+
raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
94
102
|
if @security
|
95
103
|
if @security.user_auth && @security.users.empty?
|
96
104
|
raise Fluent::ConfigError, "<user> sections required if user_auth enabled"
|
@@ -131,44 +139,35 @@ module Fluent::Plugin
|
|
131
139
|
@lsock = @usock = nil
|
132
140
|
end
|
133
141
|
|
142
|
+
HEARTBEAT_UDP_PAYLOAD = "\0"
|
143
|
+
|
134
144
|
def start
|
135
145
|
super
|
136
146
|
|
137
|
-
|
138
|
-
|
139
|
-
|
147
|
+
server_create_connection(
|
148
|
+
:in_forward_server, @port,
|
149
|
+
bind: @bind,
|
150
|
+
shared: false,
|
151
|
+
resolve_name: @resolve_hostname,
|
152
|
+
linger_timeout: @linger_timeout,
|
153
|
+
backlog: @backlog,
|
154
|
+
&method(:handle_connection)
|
155
|
+
)
|
156
|
+
|
157
|
+
server_create(:in_forward_server_udp_heartbeat, @port, shared: false, proto: :udp, bind: @bind, resolve_name: @resolve_hostname, max_bytes: 128) do |data, sock|
|
158
|
+
log.trace "heartbeat udp data arrived", host: sock.remote_host, port: sock.remote_port, data: data
|
159
|
+
begin
|
160
|
+
sock.write HEARTBEAT_UDP_PAYLOAD
|
161
|
+
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
|
162
|
+
log.trace "error while heartbeat response", host: sock.remote_host, error: e
|
163
|
+
end
|
140
164
|
end
|
141
|
-
client = ServerEngine::SocketManager::Client.new(socket_manager_path)
|
142
|
-
|
143
|
-
@lsock = listen(client)
|
144
|
-
event_loop_attach(@lsock)
|
145
|
-
|
146
|
-
@usock = client.listen_udp(@bind, @port)
|
147
|
-
@usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
|
148
|
-
@hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
|
149
|
-
event_loop_attach(@hbr)
|
150
|
-
end
|
151
|
-
|
152
|
-
def close
|
153
|
-
@lsock.close if @lsock
|
154
|
-
@usock.close if @usock
|
155
|
-
super
|
156
165
|
end
|
157
166
|
|
158
|
-
def listen(client)
|
159
|
-
log.info "listening fluent socket on #{@bind}:#{@port}"
|
160
|
-
sock = client.listen_tcp(@bind, @port)
|
161
|
-
s = Coolio::TCPServer.new(sock, nil, Handler, @linger_timeout, log, method(:handle_connection))
|
162
|
-
s.listen(@backlog) unless @backlog.nil?
|
163
|
-
s
|
164
|
-
end
|
165
|
-
|
166
|
-
private
|
167
|
-
|
168
167
|
def handle_connection(conn)
|
169
168
|
send_data = ->(serializer, data){ conn.write serializer.call(data) }
|
170
169
|
|
171
|
-
log.trace "connected fluent socket",
|
170
|
+
log.trace "connected fluent socket", addr: conn.remote_addr, port: conn.remote_port
|
172
171
|
state = :established
|
173
172
|
nonce = nil
|
174
173
|
user_auth_salt = nil
|
@@ -182,7 +181,7 @@ module Fluent::Plugin
|
|
182
181
|
state = :pingpong
|
183
182
|
end
|
184
183
|
|
185
|
-
log.trace "accepted fluent socket",
|
184
|
+
log.trace "accepted fluent socket", addr: conn.remote_addr, port: conn.remote_port
|
186
185
|
|
187
186
|
read_messages(conn) do |msg, chunk_size, serializer|
|
188
187
|
case state
|
@@ -198,15 +197,11 @@ module Fluent::Plugin
|
|
198
197
|
log.debug "connection established", address: conn.remote_addr, port: conn.remote_port
|
199
198
|
state = :established
|
200
199
|
when :established
|
201
|
-
options = on_message(msg, chunk_size, conn.
|
200
|
+
options = on_message(msg, chunk_size, conn.remote_host)
|
202
201
|
if options && r = response(options)
|
203
|
-
send_data.call(serializer, r)
|
204
202
|
log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
|
205
|
-
if @deny_keepalive
|
206
|
-
|
207
|
-
conn.close
|
208
|
-
end
|
209
|
-
end
|
203
|
+
conn.on_write_complete{ conn.close } if @deny_keepalive
|
204
|
+
send_data.call(serializer, r)
|
210
205
|
else
|
211
206
|
if @deny_keepalive
|
212
207
|
conn.close
|
@@ -222,7 +217,7 @@ module Fluent::Plugin
|
|
222
217
|
feeder = nil
|
223
218
|
serializer = nil
|
224
219
|
bytes = 0
|
225
|
-
conn.
|
220
|
+
conn.data do |data|
|
226
221
|
# only for first call of callback
|
227
222
|
unless feeder
|
228
223
|
first = data[0]
|
@@ -258,7 +253,7 @@ module Fluent::Plugin
|
|
258
253
|
nil
|
259
254
|
end
|
260
255
|
|
261
|
-
def on_message(msg, chunk_size,
|
256
|
+
def on_message(msg, chunk_size, remote_host)
|
262
257
|
if msg.nil?
|
263
258
|
# for future TCP heartbeat_request
|
264
259
|
return
|
@@ -266,7 +261,7 @@ module Fluent::Plugin
|
|
266
261
|
|
267
262
|
# TODO: raise an exception if broken chunk is generated by recoverable situation
|
268
263
|
unless msg.is_a?(Array)
|
269
|
-
log.warn "incoming chunk is broken:",
|
264
|
+
log.warn "incoming chunk is broken:", host: remote_host, msg: msg
|
270
265
|
return
|
271
266
|
end
|
272
267
|
|
@@ -274,10 +269,10 @@ module Fluent::Plugin
|
|
274
269
|
entries = msg[1]
|
275
270
|
|
276
271
|
if @chunk_size_limit && (chunk_size > @chunk_size_limit)
|
277
|
-
log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag,
|
272
|
+
log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: remote_host, limit: @chunk_size_limit, size: chunk_size
|
278
273
|
return
|
279
274
|
elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
|
280
|
-
log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag,
|
275
|
+
log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: remote_host, limit: @chunk_size_warn_limit, size: chunk_size
|
281
276
|
end
|
282
277
|
|
283
278
|
case entries
|
@@ -287,14 +282,14 @@ module Fluent::Plugin
|
|
287
282
|
size = (option && option['size']) || 0
|
288
283
|
es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
|
289
284
|
es = es_class.new(entries, nil, size.to_i)
|
290
|
-
es = check_and_skip_invalid_event(tag, es,
|
291
|
-
es = add_source_host(es,
|
285
|
+
es = check_and_skip_invalid_event(tag, es, remote_host) if @skip_invalid_event
|
286
|
+
es = add_source_host(es, remote_host) if @source_hostname_key
|
292
287
|
router.emit_stream(tag, es)
|
293
288
|
|
294
289
|
when Array
|
295
290
|
# Forward
|
296
291
|
es = if @skip_invalid_event
|
297
|
-
check_and_skip_invalid_event(tag, entries,
|
292
|
+
check_and_skip_invalid_event(tag, entries, remote_host)
|
298
293
|
else
|
299
294
|
es = Fluent::MultiEventStream.new
|
300
295
|
entries.each { |e|
|
@@ -306,7 +301,7 @@ module Fluent::Plugin
|
|
306
301
|
}
|
307
302
|
es
|
308
303
|
end
|
309
|
-
es = add_source_host(es,
|
304
|
+
es = add_source_host(es, remote_host) if @source_hostname_key
|
310
305
|
router.emit_stream(tag, es)
|
311
306
|
option = msg[2]
|
312
307
|
|
@@ -315,12 +310,12 @@ module Fluent::Plugin
|
|
315
310
|
time = msg[1]
|
316
311
|
record = msg[2]
|
317
312
|
if @skip_invalid_event && invalid_event?(tag, time, record)
|
318
|
-
log.warn "got invalid event and drop it:",
|
313
|
+
log.warn "got invalid event and drop it:", host: remote_host, tag: tag, time: time, record: record
|
319
314
|
return msg[3] # retry never succeeded so return ack and drop incoming event.
|
320
315
|
end
|
321
316
|
return if record.nil?
|
322
317
|
time = Fluent::Engine.now if time.to_i == 0
|
323
|
-
record[@source_hostname_key] =
|
318
|
+
record[@source_hostname_key] = remote_host if @source_hostname_key
|
324
319
|
router.emit(tag, time, record)
|
325
320
|
option = msg[3]
|
326
321
|
end
|
@@ -333,11 +328,11 @@ module Fluent::Plugin
|
|
333
328
|
!((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
|
334
329
|
end
|
335
330
|
|
336
|
-
def check_and_skip_invalid_event(tag, es,
|
331
|
+
def check_and_skip_invalid_event(tag, es, remote_host)
|
337
332
|
new_es = Fluent::MultiEventStream.new
|
338
333
|
es.each { |time, record|
|
339
334
|
if invalid_event?(tag, time, record)
|
340
|
-
log.warn "skip invalid event:",
|
335
|
+
log.warn "skip invalid event:", host: remote_host, tag: tag, time: time, record: record
|
341
336
|
next
|
342
337
|
end
|
343
338
|
new_es.add(time, record)
|
@@ -354,11 +349,6 @@ module Fluent::Plugin
|
|
354
349
|
new_es
|
355
350
|
end
|
356
351
|
|
357
|
-
def source_message(peeraddr)
|
358
|
-
_, port, host, addr = peeraddr
|
359
|
-
"host: #{host}, addr: #{addr}, port: #{port}"
|
360
|
-
end
|
361
|
-
|
362
352
|
def select_authenticate_users(node, username)
|
363
353
|
if node.nil? || node[:users].empty?
|
364
354
|
@security.users.select{|u| u.username == username}
|
@@ -424,114 +414,5 @@ module Fluent::Plugin
|
|
424
414
|
shared_key_digest_hex = Digest::SHA512.new.update(reason_or_salt).update(@security.self_hostname).update(nonce).update(shared_key).hexdigest
|
425
415
|
['PONG', true, '', @security.self_hostname, shared_key_digest_hex]
|
426
416
|
end
|
427
|
-
|
428
|
-
class Handler < Coolio::Socket
|
429
|
-
attr_reader :protocol, :remote_port, :remote_addr, :remote_host
|
430
|
-
|
431
|
-
PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
|
432
|
-
|
433
|
-
def initialize(io, linger_timeout, log, on_connect_callback)
|
434
|
-
super(io)
|
435
|
-
|
436
|
-
@peeraddr = nil
|
437
|
-
if io.is_a?(TCPSocket) # for unix domain socket support in the future
|
438
|
-
@peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
|
439
|
-
opt = [1, linger_timeout].pack('I!I!') # { int l_onoff; int l_linger; }
|
440
|
-
io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
441
|
-
end
|
442
|
-
|
443
|
-
### TODO: disabling name rev resolv
|
444
|
-
proto, port, host, addr = ( io.peeraddr rescue PEERADDR_FAILED )
|
445
|
-
if addr == '?'
|
446
|
-
port, addr = *Socket.unpack_sockaddr_in(io.getpeername) rescue nil
|
447
|
-
end
|
448
|
-
@protocol = proto
|
449
|
-
@remote_port = port
|
450
|
-
@remote_addr = addr
|
451
|
-
@remote_host = host
|
452
|
-
@writing = false
|
453
|
-
@closing = false
|
454
|
-
@mutex = Mutex.new
|
455
|
-
|
456
|
-
@chunk_counter = 0
|
457
|
-
@on_connect_callback = on_connect_callback
|
458
|
-
@log = log
|
459
|
-
@log.trace {
|
460
|
-
begin
|
461
|
-
remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername)
|
462
|
-
rescue
|
463
|
-
remote_port = nil
|
464
|
-
remote_addr = nil
|
465
|
-
end
|
466
|
-
[ "accepted fluent socket", {address: remote_addr, port: remote_port, instance: self.object_id} ]
|
467
|
-
}
|
468
|
-
end
|
469
|
-
|
470
|
-
def on_connect
|
471
|
-
@on_connect_callback.call(self)
|
472
|
-
end
|
473
|
-
|
474
|
-
# API to register callback for data arrival
|
475
|
-
def on_data(&callback)
|
476
|
-
@on_read_callback = callback
|
477
|
-
end
|
478
|
-
|
479
|
-
def on_read(data)
|
480
|
-
@on_read_callback.call(data)
|
481
|
-
rescue => e
|
482
|
-
@log.error "unexpected error on reading data from client", address: @remote_addr, error: e
|
483
|
-
@log.error_backtrace
|
484
|
-
close
|
485
|
-
end
|
486
|
-
|
487
|
-
def on_write_complete
|
488
|
-
closing = @mutex.synchronize {
|
489
|
-
@writing = false
|
490
|
-
@closing
|
491
|
-
}
|
492
|
-
if closing
|
493
|
-
close
|
494
|
-
end
|
495
|
-
end
|
496
|
-
|
497
|
-
def close
|
498
|
-
writing = @mutex.synchronize {
|
499
|
-
@closing = true
|
500
|
-
@writing
|
501
|
-
}
|
502
|
-
unless writing
|
503
|
-
super
|
504
|
-
end
|
505
|
-
end
|
506
|
-
end
|
507
|
-
|
508
|
-
class HeartbeatRequestHandler < Coolio::IO
|
509
|
-
def initialize(io, callback)
|
510
|
-
super(io)
|
511
|
-
@io = io
|
512
|
-
@callback = callback
|
513
|
-
end
|
514
|
-
|
515
|
-
def on_readable
|
516
|
-
begin
|
517
|
-
msg, addr = @io.recvfrom(1024)
|
518
|
-
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
|
519
|
-
return
|
520
|
-
end
|
521
|
-
host = addr[3]
|
522
|
-
port = addr[1]
|
523
|
-
@callback.call(host, port, msg)
|
524
|
-
rescue
|
525
|
-
# TODO log?
|
526
|
-
end
|
527
|
-
end
|
528
|
-
|
529
|
-
def on_heartbeat_request(host, port, msg)
|
530
|
-
#log.trace "heartbeat request from #{host}:#{port}"
|
531
|
-
begin
|
532
|
-
@usock.send "\0", 0, host, port
|
533
|
-
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
|
534
|
-
end
|
535
|
-
end
|
536
417
|
end
|
537
418
|
end
|