fluentd 1.6.3 → 1.7.0.rc1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +35 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +2 -0
  4. data/README.md +5 -1
  5. data/fluentd.gemspec +1 -1
  6. data/lib/fluent/clock.rb +4 -0
  7. data/lib/fluent/compat/output.rb +3 -3
  8. data/lib/fluent/compat/socket_util.rb +1 -1
  9. data/lib/fluent/config/element.rb +3 -3
  10. data/lib/fluent/config/literal_parser.rb +1 -1
  11. data/lib/fluent/config/section.rb +4 -1
  12. data/lib/fluent/error.rb +4 -0
  13. data/lib/fluent/event.rb +28 -24
  14. data/lib/fluent/event_router.rb +2 -1
  15. data/lib/fluent/log.rb +1 -1
  16. data/lib/fluent/msgpack_factory.rb +8 -0
  17. data/lib/fluent/plugin/bare_output.rb +4 -4
  18. data/lib/fluent/plugin/buf_file_single.rb +211 -0
  19. data/lib/fluent/plugin/buffer.rb +62 -63
  20. data/lib/fluent/plugin/buffer/chunk.rb +21 -3
  21. data/lib/fluent/plugin/buffer/file_chunk.rb +37 -12
  22. data/lib/fluent/plugin/buffer/file_single_chunk.rb +314 -0
  23. data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -1
  24. data/lib/fluent/plugin/compressable.rb +10 -6
  25. data/lib/fluent/plugin/filter_grep.rb +2 -2
  26. data/lib/fluent/plugin/formatter_csv.rb +10 -6
  27. data/lib/fluent/plugin/in_syslog.rb +10 -3
  28. data/lib/fluent/plugin/in_tail.rb +7 -2
  29. data/lib/fluent/plugin/in_tcp.rb +34 -7
  30. data/lib/fluent/plugin/multi_output.rb +4 -4
  31. data/lib/fluent/plugin/out_exec_filter.rb +1 -0
  32. data/lib/fluent/plugin/out_file.rb +13 -3
  33. data/lib/fluent/plugin/out_forward.rb +126 -588
  34. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  35. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  36. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  37. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  38. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +121 -0
  39. data/lib/fluent/plugin/out_forward/load_balancer.rb +111 -0
  40. data/lib/fluent/plugin/out_forward/socket_cache.rb +138 -0
  41. data/lib/fluent/plugin/out_http.rb +231 -0
  42. data/lib/fluent/plugin/output.rb +29 -35
  43. data/lib/fluent/plugin/parser.rb +77 -0
  44. data/lib/fluent/plugin/parser_csv.rb +75 -0
  45. data/lib/fluent/plugin_helper/server.rb +1 -1
  46. data/lib/fluent/plugin_helper/thread.rb +1 -0
  47. data/lib/fluent/root_agent.rb +1 -1
  48. data/lib/fluent/time.rb +4 -2
  49. data/lib/fluent/timezone.rb +21 -7
  50. data/lib/fluent/version.rb +1 -1
  51. data/test/command/test_fluentd.rb +1 -1
  52. data/test/command/test_plugin_generator.rb +18 -2
  53. data/test/config/test_configurable.rb +78 -40
  54. data/test/counter/test_store.rb +1 -1
  55. data/test/helper.rb +1 -0
  56. data/test/helpers/process_extenstion.rb +33 -0
  57. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  58. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  59. data/test/plugin/out_forward/test_handshake_protocol.rb +103 -0
  60. data/test/plugin/out_forward/test_load_balancer.rb +60 -0
  61. data/test/plugin/out_forward/test_socket_cache.rb +139 -0
  62. data/test/plugin/test_buf_file.rb +118 -2
  63. data/test/plugin/test_buf_file_single.rb +734 -0
  64. data/test/plugin/test_buffer.rb +4 -48
  65. data/test/plugin/test_buffer_file_chunk.rb +19 -1
  66. data/test/plugin/test_buffer_file_single_chunk.rb +620 -0
  67. data/test/plugin/test_formatter_csv.rb +16 -0
  68. data/test/plugin/test_in_syslog.rb +56 -6
  69. data/test/plugin/test_in_tail.rb +1 -1
  70. data/test/plugin/test_in_tcp.rb +25 -0
  71. data/test/plugin/test_out_forward.rb +75 -201
  72. data/test/plugin/test_out_http.rb +352 -0
  73. data/test/plugin/test_output_as_buffered.rb +27 -24
  74. data/test/plugin/test_parser.rb +40 -0
  75. data/test/plugin/test_parser_csv.rb +83 -0
  76. data/test/plugin_helper/test_record_accessor.rb +1 -1
  77. data/test/test_time_formatter.rb +140 -121
  78. metadata +35 -6
@@ -0,0 +1,161 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+ require 'fluent/plugin_helper/socket'
19
+ require 'fluent/engine'
20
+ require 'fluent/clock'
21
+
22
+ module Fluent::Plugin
23
+ class ForwardOutput < Output
24
+ class AckHandler
25
+ module Result
26
+ SUCCESS = :success
27
+ FAILED = :failed
28
+ CHUNKID_UNMATCHED = :chunkid_unmatched
29
+ end
30
+
31
+ def initialize(timeout:, log:, read_length:)
32
+ @mutex = Mutex.new
33
+ @ack_waitings = []
34
+ @timeout = timeout
35
+ @log = log
36
+ @read_length = read_length
37
+ @unpacker = Fluent::Engine.msgpack_unpacker
38
+ end
39
+
40
+ def collect_response(select_interval)
41
+ now = Fluent::Clock.now
42
+ sockets = []
43
+ results = []
44
+ begin
45
+ new_list = []
46
+ @mutex.synchronize do
47
+ @ack_waitings.each do |info|
48
+ if info.expired?(now)
49
+ # There are 2 types of cases when no response has been received from socket:
50
+ # (1) the node does not support sending responses
51
+ # (2) the node does support sending response but responses have not arrived for some reasons.
52
+ @log.warn 'no response from node. regard it as unavailable.', host: info.node.host, port: info.node.port
53
+ results << [info, Result::FAILED]
54
+ else
55
+ sockets << info.sock
56
+ new_list << info
57
+ end
58
+ end
59
+ @ack_waitings = new_list
60
+ end
61
+
62
+ readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
63
+ if readable_sockets
64
+ readable_sockets.each do |sock|
65
+ results << read_ack_from_sock(sock)
66
+ end
67
+ end
68
+
69
+ results.each do |info, ret|
70
+ if info.nil?
71
+ yield nil, nil, nil, ret
72
+ else
73
+ yield info.chunk_id, info.node, info.sock, ret
74
+ end
75
+ end
76
+ rescue => e
77
+ @log.error 'unexpected error while receiving ack', error: e
78
+ @log.error_backtrace
79
+ end
80
+ end
81
+
82
+ ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time) do
83
+ def expired?(now)
84
+ expired_time < now
85
+ end
86
+ end
87
+
88
+ Ack = Struct.new(:chunk_id, :node, :handler) do
89
+ def enqueue(sock)
90
+ handler.enqueue(node, sock, chunk_id)
91
+ end
92
+ end
93
+
94
+ def create_ack(chunk_id, node)
95
+ Ack.new(chunk_id, node, self)
96
+ end
97
+
98
+ def enqueue(node, sock, cid)
99
+ info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout)
100
+ @mutex.synchronize do
101
+ @ack_waitings << info
102
+ end
103
+ end
104
+
105
+ private
106
+
107
+ def read_ack_from_sock(sock)
108
+ begin
109
+ raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
110
+ rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
111
+ raw_data = ''
112
+ end
113
+
114
+ info = find(sock)
115
+
116
+ # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
117
+ # If this happens we assume the data wasn't delivered and retry it.
118
+ if raw_data.empty?
119
+ @log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
120
+ # info.node.disable!
121
+ return info, Result::FAILED
122
+ else
123
+ @unpacker.feed(raw_data)
124
+ res = @unpacker.read
125
+ @log.trace 'getting response from destination', host: info.node.host, port: info.node.port, chunk_id: dump_unique_id_hex(info.chunk_id), response: res
126
+ if res['ack'] != info.chunk_id_base64
127
+ # Some errors may have occurred when ack and chunk id is different, so send the chunk again.
128
+ @log.warn 'ack in response and chunk id in sent data are different', chunk_id: dump_unique_id_hex(info.chunk_id), ack: res['ack']
129
+ return info, Result::CHUNKID_UNMATCHED
130
+ else
131
+ @log.trace 'got a correct ack response', chunk_id: dump_unique_id_hex(info.chunk_id)
132
+ end
133
+
134
+ return info, Result::SUCCESS
135
+ end
136
+ rescue => e
137
+ @log.error 'unexpected error while receiving ack message', error: e
138
+ @log.error_backtrace
139
+ [nil, Result::FAILED]
140
+ ensure
141
+ delete(info)
142
+ end
143
+
144
+ def dump_unique_id_hex(unique_id)
145
+ Fluent::UniqueId.hex(unique_id)
146
+ end
147
+
148
+ def find(sock)
149
+ @mutex.synchronize do
150
+ @ack_waitings.find { |info| info.sock == sock }
151
+ end
152
+ end
153
+
154
+ def delete(info)
155
+ @mutex.synchronize do
156
+ @ack_waitings.delete(info)
157
+ end
158
+ end
159
+ end
160
+ end
161
+ end
@@ -0,0 +1,113 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+
19
+ module Fluent::Plugin
20
+ class ForwardOutput < Output
21
+ class ConnectionManager
22
+ RequestInfo = Struct.new(:state, :shared_key_nonce, :auth)
23
+
24
+ # @param log [Logger]
25
+ # @param secure [Boolean]
26
+ # @param connection_factory [Proc]
27
+ # @param SocketCache [Fluent::ForwardOutput::SocketCache]
28
+ def initialize(log:, secure:, connection_factory:, socket_cache:)
29
+ @log = log
30
+ @secure = secure
31
+ @connection_factory = connection_factory
32
+ @socket_cache = socket_cache
33
+ end
34
+
35
+ def stop
36
+ @socket_cache && @socket_cache.clear
37
+ end
38
+
39
+ # @param ack [Fluent::Plugin::ForwardOutput::AckHander::Ack|nil]
40
+ def connect(host:, port:, hostname:, ack: nil, &block)
41
+ if @socket_cache
42
+ return connect_keepalive(host: host, port: port, hostname: hostname, ack: ack, &block)
43
+ end
44
+
45
+ @log.debug('connect new socket')
46
+ socket = @connection_factory.call(host, port, hostname)
47
+ request_info = RequestInfo.new(@secure ? :helo : :established)
48
+
49
+ unless block_given?
50
+ return [socket, request_info]
51
+ end
52
+
53
+ begin
54
+ yield(socket, request_info)
55
+ ensure
56
+ if ack
57
+ ack.enqueue(socket)
58
+ else
59
+ socket.close_write rescue nil
60
+ socket.close rescue nil
61
+ end
62
+ end
63
+ end
64
+
65
+ def purge_obsolete_socks
66
+ unless @socket_cache
67
+ raise "Do not call this method without keepalive option"
68
+ end
69
+ @socket_cache.purge_obsolete_socks
70
+ end
71
+
72
+ def close(sock)
73
+ if @socket_cache
74
+ @socket_cache.checkin(sock)
75
+ else
76
+ sock.close_write rescue nil
77
+ sock.close rescue nil
78
+ end
79
+ end
80
+
81
+ private
82
+
83
+ def connect_keepalive(host:, port:, hostname:, ack: nil)
84
+ request_info = RequestInfo.new(:established)
85
+ socket = @socket_cache.checkout_or([host, port, hostname]) do
86
+ s = @connection_factory.call(host, port, hostname)
87
+ request_info = RequestInfo.new(@secure ? :helo : :established) # overwrite if new connection
88
+ s
89
+ end
90
+
91
+ unless block_given?
92
+ return [socket, request_info]
93
+ end
94
+
95
+ ret = nil
96
+ begin
97
+ ret = yield(socket, request_info)
98
+ rescue
99
+ @socket_cache.revoke(socket)
100
+ raise
101
+ else
102
+ if ack
103
+ ack.enqueue(socket)
104
+ else
105
+ @socket_cache.checkin(socket)
106
+ end
107
+ end
108
+
109
+ ret
110
+ end
111
+ end
112
+ end
113
+ end
@@ -0,0 +1,28 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+
19
+ module Fluent::Plugin
20
+ class ForwardOutput < Output
21
+ class Error < StandardError; end
22
+ class NoNodesAvailable < Error; end
23
+ class ConnectionClosedError < Error; end
24
+ class HandshakeError < Error; end
25
+ class HeloError < HandshakeError; end
26
+ class PingpongError < HandshakeError; end
27
+ end
28
+ end
@@ -0,0 +1,84 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+
19
+ module Fluent::Plugin
20
+ class ForwardOutput < Output
21
+ class FailureDetector
22
+ PHI_FACTOR = 1.0 / Math.log(10.0)
23
+ SAMPLE_SIZE = 1000
24
+
25
+ def initialize(heartbeat_interval, hard_timeout, init_last)
26
+ @heartbeat_interval = heartbeat_interval
27
+ @last = init_last
28
+ @hard_timeout = hard_timeout
29
+
30
+ # microsec
31
+ @init_gap = (heartbeat_interval * 1e6).to_i
32
+ @window = [@init_gap]
33
+ end
34
+
35
+ def hard_timeout?(now)
36
+ now - @last > @hard_timeout
37
+ end
38
+
39
+ def add(now)
40
+ if @window.empty?
41
+ @window << @init_gap
42
+ @last = now
43
+ else
44
+ gap = now - @last
45
+ @window << (gap * 1e6).to_i
46
+ @window.shift if @window.length > SAMPLE_SIZE
47
+ @last = now
48
+ end
49
+ end
50
+
51
+ def phi(now)
52
+ size = @window.size
53
+ return 0.0 if size == 0
54
+
55
+ # Calculate weighted moving average
56
+ mean_usec = 0
57
+ fact = 0
58
+ @window.each_with_index {|gap,i|
59
+ mean_usec += gap * (1+i)
60
+ fact += (1+i)
61
+ }
62
+ mean_usec = mean_usec / fact
63
+
64
+ # Normalize arrive intervals into 1sec
65
+ mean = (mean_usec.to_f / 1e6) - @heartbeat_interval + 1
66
+
67
+ # Calculate phi of the phi accrual failure detector
68
+ t = now - @last - @heartbeat_interval + 1
69
+ phi = PHI_FACTOR * t / mean
70
+
71
+ return phi
72
+ end
73
+
74
+ def sample_size
75
+ @window.size
76
+ end
77
+
78
+ def clear
79
+ @window.clear
80
+ @last = 0
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,121 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+ require 'fluent/plugin/out_forward/error'
19
+ require 'digest'
20
+
21
+ module Fluent::Plugin
22
+ class ForwardOutput < Output
23
+ class HandshakeProtocol
24
+ def initialize(log:, hostname:, shared_key:, password:, username:)
25
+ @log = log
26
+ @hostname = hostname
27
+ @shared_key = shared_key
28
+ @password = password
29
+ @username = username
30
+ @shared_key_salt = generate_salt
31
+ end
32
+
33
+ def invoke(sock, ri, data)
34
+ @log.trace __callee__
35
+
36
+ case ri.state
37
+ when :helo
38
+ unless check_helo(ri, data)
39
+ raise HeloError, 'received invalid helo message'
40
+ end
41
+
42
+ sock.write(generate_ping(ri).to_msgpack)
43
+ ri.state = :pingpong
44
+ when :pingpong
45
+ succeeded, reason = check_pong(ri, data)
46
+ unless succeeded
47
+ raise PingpongError, reason
48
+ end
49
+
50
+ ri.state = :established
51
+ else
52
+ raise "BUG: unknown session state: #{ri.state}"
53
+ end
54
+ end
55
+
56
+ private
57
+
58
+ def check_pong(ri, message)
59
+ @log.debug('checking pong')
60
+ # ['PONG', bool(authentication result), 'reason if authentication failed',
61
+ # self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)]
62
+ unless message.size == 5 && message[0] == 'PONG'
63
+ return false, 'invalid format for PONG message'
64
+ end
65
+ _pong, auth_result, reason, hostname, shared_key_hexdigest = message
66
+
67
+ unless auth_result
68
+ return false, 'authentication failed: ' + reason
69
+ end
70
+
71
+ if hostname == @hostname
72
+ return false, 'same hostname between input and output: invalid configuration'
73
+ end
74
+
75
+ clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(ri.shared_key_nonce).update(@shared_key).hexdigest
76
+ unless shared_key_hexdigest == clientside
77
+ return false, 'shared key mismatch'
78
+ end
79
+
80
+ [true, nil]
81
+ end
82
+
83
+ def check_helo(ri, message)
84
+ @log.debug('checking helo')
85
+ # ['HELO', options(hash)]
86
+ unless message.size == 2 && message[0] == 'HELO'
87
+ return false
88
+ end
89
+
90
+ opts = message[1] || {}
91
+ # make shared_key_check failed (instead of error) if protocol version mismatch exist
92
+ ri.shared_key_nonce = opts['nonce'] || ''
93
+ ri.auth = opts['auth'] || ''
94
+ true
95
+ end
96
+
97
+ def generate_ping(ri)
98
+ @log.debug('generating ping')
99
+ # ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key),
100
+ # username || '', sha512\_hex(auth\_salt + username + password) || '']
101
+ shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt)
102
+ .update(@hostname)
103
+ .update(ri.shared_key_nonce)
104
+ .update(@shared_key)
105
+ .hexdigest
106
+ ping = ['PING', @hostname, @shared_key_salt, shared_key_hexdigest]
107
+ if !ri.auth.empty?
108
+ password_hexdigest = Digest::SHA512.new.update(ri.auth).update(@username).update(@password).hexdigest
109
+ ping.push(@username, password_hexdigest)
110
+ else
111
+ ping.push('', '')
112
+ end
113
+ ping
114
+ end
115
+
116
+ def generate_salt
117
+ SecureRandom.hex(16)
118
+ end
119
+ end
120
+ end
121
+ end