fluentd 1.4.2-x64-mingw32 → 1.5.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: dfbfa9c16c04eeac46d6250f4a97d5cacca03df0
4
- data.tar.gz: 5fa4cb163e1a18f67b2190d5fdc04d8da2627b6f
2
+ SHA256:
3
+ metadata.gz: f4abf060d857cfa30f804057fc677fd38ab52773b6f7816c899fd737eb42028e
4
+ data.tar.gz: 4cbe1d0d02ebe98c882581ef98c35cabcb6654db88c6184c24fdff9f823c6649
5
5
  SHA512:
6
- metadata.gz: 56d4316114cc19cab79250375ab8dc6027c938429b725f3051382a8479fc2ae960d04750ce07847c473d0a91920af25be9f1eca085c526d26c59b2a6504a8cfb
7
- data.tar.gz: dfb2a741dfd87f14ee010377197029248201b6f2dc3e91e89b410daa443e9517e6f2e7c5b1f6afd0e79dfe85010a260ca7791cd80a2680de5fa17f7df5fe7c41
6
+ metadata.gz: 5bee075a321cca5c804f7e90f1deca33c6998213daa3b0913dc0286fdb5c81199d51fdb1e0e7acfe820d2894bf844835994e9a3defc1c5e0096085ae8c214983
7
+ data.tar.gz: 2ceffd4ca2a7c35184592ea6621d857e5ef7c25c8240af3662573431979ab7d2a4daaed6e7b9da5bbe5d80752390b7dff2a91a827d4e833fdec5593a9aa9eba8
@@ -11,14 +11,20 @@ matrix:
11
11
  os: linux
12
12
  - rvm: 2.2.10
13
13
  os: linux
14
- - rvm: 2.4.5
14
+ - rvm: 2.2.10
15
+ os: linux-ppc64le
16
+ - rvm: 2.4.6
15
17
  os: linux
16
- - rvm: 2.5.3
18
+ - rvm: 2.4.6
19
+ os: linux-ppc64le
20
+ - rvm: 2.5.5
17
21
  os: linux
18
- - rvm: 2.6.0
22
+ - rvm: 2.6.3
19
23
  os: linux
20
24
  - rvm: ruby-head
21
25
  os: linux
26
+ - rvm: ruby-head
27
+ os: linux-ppc64le
22
28
  - rvm: 2.1.10
23
29
  os: osx
24
30
  osx_image: xcode8.3 # OSX 10.12
@@ -28,7 +34,7 @@ matrix:
28
34
  # - rvm: 2.3.3
29
35
  # os: osx
30
36
  # osx_image: xcode8.2 # OSX 10.12
31
- - rvm: 2.4.1
37
+ - rvm: 2.4.6
32
38
  os: osx
33
39
  osx_image: xcode8.3 # OSX 10.12
34
40
  - rvm: ruby-head
@@ -40,7 +46,7 @@ matrix:
40
46
  - rvm: 2.1.10
41
47
  os: osx
42
48
  osx_image: xcode8.3
43
- - rvm: 2.4.1
49
+ - rvm: 2.4.6
44
50
  os: osx
45
51
  osx_image: xcode8.3
46
52
  - rvm: ruby-head
@@ -1,3 +1,38 @@
1
+ # v1.5
2
+
3
+ ## Release v1.5.0 - 2019/05/18
4
+
5
+ ### New feature
6
+
7
+ * out_forward: Support keepalive feature
8
+ https://github.com/fluent/fluentd/pull/2393
9
+ * in_http: Support TLS via server helper
10
+ https://github.com/fluent/fluentd/pull/2395
11
+ * in_syslog: Support TLS via server helper
12
+ https://github.com/fluent/fluentd/pull/2399
13
+
14
+ ### Enhancement
15
+
16
+ * in_syslog: Add delimiter parameter
17
+ https://github.com/fluent/fluentd/pull/2378
18
+ * in_forward: Add tag/add_tag_prefix parameters
19
+ https://github.com/fluent/fluentd/pull/2396
20
+ * parser_json: Add stream_buffer_size parameter for yajl
21
+ https://github.com/fluent/fluentd/pull/2381
22
+ * command: Add deprecated message to show-plugin-config option
23
+ https://github.com/fluent/fluentd/pull/2401
24
+ * storage_local: Ignore empty file. Call sync after write for XFS.
25
+ https://github.com/fluent/fluentd/pull/2409
26
+
27
+ ### Bug fixes
28
+
29
+ * out_forward: Don't use SO_LINGER on SSL/TLS WinSock
30
+ https://github.com/fluent/fluentd/pull/2398
31
+ * server helper: Fix recursive lock issue in TLSServer
32
+ https://github.com/fluent/fluentd/pull/2341
33
+ * Fix typo
34
+ https://github.com/fluent/fluentd/pull/2369
35
+
1
36
  # v1.4
2
37
 
3
38
  ## Release v1.4.2 - 2019/04/02
data/README.md CHANGED
@@ -30,6 +30,11 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
30
30
 
31
31
  ## Development
32
32
 
33
+ ### Branch
34
+
35
+ - master: For v1 development.
36
+ - v0.12: For v0.12. This is security maintenance mode. Only security fix is accepted.
37
+
33
38
  ### Prerequisites
34
39
 
35
40
  - Ruby 2.1 or later
@@ -6,7 +6,9 @@
6
6
  keepalive_timeout 10
7
7
  # backlog 0
8
8
  add_http_headers false
9
- format default
9
+ <parse>
10
+ @type json
11
+ </parse>
10
12
  </source>
11
13
 
12
14
  <match test>
@@ -40,7 +40,7 @@ op.on('--dry-run', "Check fluentd setup is correct or not", TrueClass) {|b|
40
40
  opts[:dry_run] = b
41
41
  }
42
42
 
43
- op.on('--show-plugin-config=PLUGIN', "Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
43
+ op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
44
44
  opts[:show_plugin_config] = plugin
45
45
  }
46
46
 
@@ -148,7 +148,7 @@ module Fluent
148
148
  def on_message(msg, addr)
149
149
  @parser.parse(msg) { |time, record|
150
150
  unless time && record
151
- log.warn "pattern not match: #{msg.inspect}"
151
+ log.warn "pattern not matched: #{msg.inspect}"
152
152
  return
153
153
  end
154
154
 
@@ -148,10 +148,10 @@ module Fluent
148
148
  end
149
149
 
150
150
  attr_reader :format
151
+ attr_reader :time_format
151
152
  attr_accessor :log_event_enabled
152
153
  attr_accessor :out
153
154
  attr_accessor :level
154
- attr_accessor :time_format
155
155
  attr_accessor :optional_header, :optional_attrs
156
156
 
157
157
  def logdev=(logdev)
@@ -82,7 +82,7 @@ module Fluent::Plugin
82
82
  return t, r
83
83
  else
84
84
  if @emit_invalid_record_to_error
85
- router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not match with data '#{raw_value}'"))
85
+ router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
86
86
  end
87
87
  if @reserve_data
88
88
  t = time
@@ -62,6 +62,11 @@ module Fluent::Plugin
62
62
  desc "The field name of the client's hostname."
63
63
  config_param :source_hostname_key, :string, default: nil
64
64
 
65
+ desc "New tag instead of incoming tag"
66
+ config_param :tag, :string, default: nil
67
+ desc "Add prefix to incoming tag"
68
+ config_param :add_tag_prefix, :string, default: nil
69
+
65
70
  config_section :security, required: false, multi: false do
66
71
  desc 'The hostname'
67
72
  config_param :self_hostname, :string
@@ -106,6 +111,9 @@ module Fluent::Plugin
106
111
  end
107
112
  @enable_field_injection = @source_address_key || @source_hostname_key
108
113
 
114
+ raise Fluent::ConfigError, "'tag' parameter must not be empty" if @tag && @tag.empty?
115
+ raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
116
+
109
117
  if @security
110
118
  if @security.user_auth && @security.users.empty?
111
119
  raise Fluent::ConfigError, "<user> sections required if user_auth enabled"
@@ -293,6 +301,9 @@ module Fluent::Plugin
293
301
  log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: conn.remote_host, limit: @chunk_size_warn_limit, size: chunk_size
294
302
  end
295
303
 
304
+ tag = @tag.dup if @tag
305
+ tag = "#{@add_tag_prefix}.#{tag}" if @add_tag_prefix
306
+
296
307
  case entries
297
308
  when String
298
309
  # PackedForward
@@ -36,9 +36,7 @@ module Fluent::Plugin
36
36
  class HttpInput < Input
37
37
  Fluent::Plugin.register_input('http', self)
38
38
 
39
- # TODO: update this plugin implementation to use server plugin helper, after adding keepalive feature on it
40
-
41
- helpers :parser, :compat_parameters, :event_loop
39
+ helpers :parser, :compat_parameters, :event_loop, :server
42
40
 
43
41
  EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
44
42
 
@@ -130,28 +128,15 @@ module Fluent::Plugin
130
128
 
131
129
  log.debug "listening http", bind: @bind, port: @port
132
130
 
133
- socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
134
- if Fluent.windows?
135
- socket_manager_path = socket_manager_path.to_i
136
- end
137
- client = ServerEngine::SocketManager::Client.new(socket_manager_path)
138
- lsock = client.listen_tcp(@bind, @port)
139
-
140
131
  @km = KeepaliveManager.new(@keepalive_timeout)
141
- @lsock = Coolio::TCPServer.new(
142
- lsock, nil, Handler, @km, method(:on_request),
143
- @body_size_limit, @format_name, log,
144
- @cors_allow_origins
145
- )
146
- @lsock.listen(@backlog) unless @backlog.nil?
147
132
  event_loop_attach(@km)
148
- event_loop_attach(@lsock)
149
133
 
134
+ server_create_connection(:in_http, @port, bind: @bind, backlog: @backlog, &method(:on_server_connect))
150
135
  @float_time_parser = Fluent::NumericTimeParser.new(:float)
151
136
  end
152
137
 
153
138
  def close
154
- @lsock.close
139
+ server_wait_until_stop
155
140
  super
156
141
  end
157
142
 
@@ -240,6 +225,22 @@ module Fluent::Plugin
240
225
 
241
226
  private
242
227
 
228
+ def on_server_connect(conn)
229
+ handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins)
230
+
231
+ conn.on(:data) do |data|
232
+ handler.on_read(data)
233
+ end
234
+
235
+ conn.on(:write_complete) do |_|
236
+ handler.on_write_complete
237
+ end
238
+
239
+ conn.on(:close) do |_|
240
+ handler.on_close
241
+ end
242
+ end
243
+
243
244
  def parse_params_default(params)
244
245
  if msgpack = params['msgpack']
245
246
  @parser_msgpack.parse(msgpack) do |_time, record|
@@ -265,11 +266,11 @@ module Fluent::Plugin
265
266
  end
266
267
  end
267
268
 
268
- class Handler < Coolio::Socket
269
+ class Handler
269
270
  attr_reader :content_type
270
271
 
271
272
  def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
272
- super(io)
273
+ @io = io
273
274
  @km = km
274
275
  @callback = callback
275
276
  @body_size_limit = body_size_limit
@@ -280,7 +281,8 @@ module Fluent::Plugin
280
281
  @idle = 0
281
282
  @km.add(self)
282
283
 
283
- @remote_port, @remote_addr = *Socket.unpack_sockaddr_in(io.getpeername) rescue nil
284
+ @remote_port, @remote_addr = io.remote_port, io.remote_addr
285
+ @parser = Http::Parser.new(self)
284
286
  end
285
287
 
286
288
  def step_idle
@@ -291,17 +293,13 @@ module Fluent::Plugin
291
293
  @km.delete(self)
292
294
  end
293
295
 
294
- def on_connect
295
- @parser = Http::Parser.new(self)
296
- end
297
-
298
296
  def on_read(data)
299
297
  @idle = 0
300
298
  @parser << data
301
299
  rescue
302
300
  @log.warn "unexpected error", error: $!.to_s
303
301
  @log.warn_backtrace
304
- close
302
+ @io.close
305
303
  end
306
304
 
307
305
  def on_message_begin
@@ -477,8 +475,12 @@ module Fluent::Plugin
477
475
  end
478
476
  end
479
477
 
478
+ def close
479
+ @io.close
480
+ end
481
+
480
482
  def on_write_complete
481
- close if @next_close
483
+ @io.close if @next_close
482
484
  end
483
485
 
484
486
  def send_response_and_close(code, header, body)
@@ -499,9 +501,9 @@ module Fluent::Plugin
499
501
  data << "#{k}: #{v}\r\n"
500
502
  }
501
503
  data << "\r\n"
502
- write data
504
+ @io.write(data)
503
505
 
504
- write body
506
+ @io.write(body)
505
507
  end
506
508
 
507
509
  def send_response_nobody(code, header)
@@ -510,7 +512,7 @@ module Fluent::Plugin
510
512
  data << "#{k}: #{v}\r\n"
511
513
  }
512
514
  data << "\r\n"
513
- write data
515
+ @io.write(data)
514
516
  end
515
517
 
516
518
  def include_cors_allow_origin
@@ -74,7 +74,7 @@ module Fluent::Plugin
74
74
  desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
75
75
  config_param :tag, :string
76
76
  desc 'The transport protocol used to receive logs.(udp, tcp)'
77
- config_param :protocol_type, :enum, list: [:tcp, :udp], default: :udp
77
+ config_param :protocol_type, :enum, list: [:tcp, :udp], default: nil, deprecated: "use transport directive"
78
78
  desc 'The message frame type.(traditional, octet_count)'
79
79
  config_param :frame_type, :enum, list: [:traditional, :octet_count], default: :traditional
80
80
 
@@ -99,11 +99,19 @@ module Fluent::Plugin
99
99
 
100
100
  config_param :blocking_timeout, :time, default: 0.5
101
101
 
102
+ desc 'The delimiter value "\n"'
103
+ config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message
104
+
102
105
  config_section :parse do
103
106
  config_set_default :@type, DEFAULT_PARSER
104
107
  config_param :with_priority, :bool, default: true
105
108
  end
106
109
 
110
+ # overwrite server plugin to change default to :udp
111
+ config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
112
+ config_argument :protocol, :enum, list: [:tcp, :udp, :tls], default: :udp
113
+ end
114
+
107
115
  def configure(conf)
108
116
  compat_parameters_convert(conf, :parser)
109
117
 
@@ -138,12 +146,13 @@ module Fluent::Plugin
138
146
  def start
139
147
  super
140
148
 
141
- log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
142
- case @protocol_type
149
+ log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type || @transport_config.protocol}"
150
+ case @protocol_type || @transport_config.protocol
143
151
  when :udp then start_udp_server
144
152
  when :tcp then start_tcp_server
153
+ when :tls then start_tcp_server(tls: true)
145
154
  else
146
- raise "BUG: invalid protocol_type value:#{@protocol_type}"
155
+ raise "BUG: invalid transport value: #{@protocol_type || @transport_config.protocol}"
147
156
  end
148
157
  end
149
158
 
@@ -153,13 +162,12 @@ module Fluent::Plugin
153
162
  end
154
163
  end
155
164
 
156
- def start_tcp_server
165
+ def start_tcp_server(tls: false)
157
166
  octet_count_frame = @frame_type == :octet_count
158
167
 
159
- # syslog family adds "\n" to each message when transport is TCP and traditional frame
160
- delimiter = octet_count_frame ? " " : "\n"
168
+ delimiter = octet_count_frame ? " " : @delimiter
161
169
  delimiter_size = delimiter.size
162
- server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
170
+ server_create_connection(tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
163
171
  conn.data do |data|
164
172
  buffer = conn.buffer
165
173
  buffer << data
@@ -427,7 +427,7 @@ module Fluent::Plugin
427
427
  record[@path_key] ||= tail_watcher.path unless @path_key.nil?
428
428
  es.add(Fluent::EventTime.now, record)
429
429
  end
430
- log.warn "pattern not match: #{line.inspect}"
430
+ log.warn "pattern not matched: #{line.inspect}"
431
431
  end
432
432
  }
433
433
  rescue => e
@@ -71,7 +71,7 @@ module Fluent::Plugin
71
71
 
72
72
  @parser.parse(msg) do |time, record|
73
73
  unless time && record
74
- log.warn "pattern not match", message: msg
74
+ log.warn "pattern not matched", message: msg
75
75
  next
76
76
  end
77
77
 
@@ -74,7 +74,7 @@ module Fluent::Plugin
74
74
  begin
75
75
  @parser.parse(data) do |time, record|
76
76
  unless time && record
77
- log.warn "pattern not match", data: data
77
+ log.warn "pattern not matched", data: data
78
78
  next
79
79
  end
80
80
 
@@ -95,6 +95,7 @@ module Fluent::Plugin
95
95
  COMPAT_PARSE_PARAMS = {
96
96
  'out_format' => '@type',
97
97
  'out_keys' => 'keys',
98
+ 'out_stream_buffer_size' => 'stream_buffer_size',
98
99
  }
99
100
  COMPAT_EXTRACT_PARAMS = {
100
101
  'out_tag_key' => 'tag_key',
@@ -36,7 +36,6 @@ module Fluent::Plugin
36
36
  desc 'The transport protocol.'
37
37
  config_param :transport, :enum, list: [:tcp, :tls], default: :tcp
38
38
  # TODO: TLS session cache/tickets
39
- # TODO: Connection keepalive
40
39
 
41
40
  desc 'The timeout time when sending event logs.'
42
41
  config_param :send_timeout, :time, default: 60
@@ -103,6 +102,10 @@ module Fluent::Plugin
103
102
  config_param :tls_client_private_key_path, :string, default: nil
104
103
  desc 'The client private key passphrase for TLS.'
105
104
  config_param :tls_client_private_key_passphrase, :string, default: nil, secret: true
105
+ desc "Enable keepalive connection."
106
+ config_param :keepalive, :bool, default: false
107
+ desc "Expired time of keepalive. Default value is nil, which means to keep connection as long as possible"
108
+ config_param :keepalive_timeout, :time, default: nil
106
109
 
107
110
  config_section :security, required: false, multi: false do
108
111
  desc 'The hostname'
@@ -151,6 +154,7 @@ module Fluent::Plugin
151
154
  @usock = nil
152
155
  @sock_ack_waiting = nil
153
156
  @sock_ack_waiting_mutex = nil
157
+ @keep_alive_watcher_interval = 5 # TODO
154
158
  end
155
159
 
156
160
  def configure(conf)
@@ -201,9 +205,9 @@ module Fluent::Plugin
201
205
 
202
206
  log.info "adding forwarding server '#{name}'", host: server.host, port: server.port, weight: server.weight, plugin_id: plugin_id
203
207
  if @heartbeat_type == :none
204
- @nodes << NoneHeartbeatNode.new(self, server, failure: failure)
208
+ @nodes << NoneHeartbeatNode.new(self, server, failure: failure, keepalive: @keepalive, keepalive_timeout: @keepalive_timeout)
205
209
  else
206
- node = Node.new(self, server, failure: failure)
210
+ node = Node.new(self, server, failure: failure, keepalive: @keepalive, keepalive_timeout: @keepalive_timeout)
207
211
  begin
208
212
  node.validate_host_resolution!
209
213
  rescue => e
@@ -227,6 +231,10 @@ module Fluent::Plugin
227
231
  raise Fluent::ConfigError, "forward output plugin requires at least one <server> is required"
228
232
  end
229
233
 
234
+ if !@keepalive && @keepalive_timeout
235
+ log.warn('The value of keepalive_timeout is ignored. if you want to use keepalive, please add `keepalive true` to your conf.')
236
+ end
237
+
230
238
  raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
231
239
  end
232
240
 
@@ -279,6 +287,10 @@ module Fluent::Plugin
279
287
  end
280
288
  end
281
289
  end
290
+
291
+ if @keepalive && @keepalive_timeout
292
+ timer_execute(:out_forward_keep_alived_socket_watcher, @keep_alive_watcher_interval, &method(:on_purge_obsolete_socks))
293
+ end
282
294
  end
283
295
 
284
296
  def close
@@ -286,6 +298,10 @@ module Fluent::Plugin
286
298
  # close socket and ignore errors: this socket will not be used anyway.
287
299
  @usock.close rescue nil
288
300
  end
301
+
302
+ if @keepalive && @keepalive_timeout
303
+ @nodes.each(&:clear)
304
+ end
289
305
  super
290
306
  end
291
307
 
@@ -354,7 +370,10 @@ module Fluent::Plugin
354
370
  cert_path: @tls_client_cert_path,
355
371
  private_key_path: @tls_client_private_key_path,
356
372
  private_key_passphrase: @tls_client_private_key_passphrase,
357
- linger_timeout: @send_timeout,
373
+
374
+ # Enabling SO_LINGER causes data loss on Windows
375
+ # https://github.com/fluent/fluentd/issues/1968
376
+ linger_timeout: Fluent.windows? ? nil : @send_timeout,
358
377
  send_timeout: @send_timeout,
359
378
  recv_timeout: @ack_response_timeout,
360
379
  &block
@@ -450,6 +469,10 @@ module Fluent::Plugin
450
469
  end
451
470
  end
452
471
 
472
+ def on_purge_obsolete_socks
473
+ @nodes.each(&:purge_obsolete_socks)
474
+ end
475
+
453
476
  # return chunk id to be committed
454
477
  def read_ack_from_sock(sock, unpacker)
455
478
  begin
@@ -484,8 +507,13 @@ module Fluent::Plugin
484
507
  log.error "unexpected error while receiving ack message", error: e
485
508
  log.error_backtrace
486
509
  ensure
487
- info.sock.close_write rescue nil
488
- info.sock.close rescue nil
510
+ if @keepalive
511
+ info.node.socket_cache.dec_ref_by_value(info.sock)
512
+ else
513
+ info.sock.close_write rescue nil
514
+ info.sock.close rescue nil
515
+ end
516
+
489
517
  @sock_ack_waiting_mutex.synchronize do
490
518
  @sock_ack_waiting.delete(info)
491
519
  end
@@ -513,6 +541,9 @@ module Fluent::Plugin
513
541
  # (2) the node does support sending response but responses have not arrived for some reasons.
514
542
  log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
515
543
  info.node.disable!
544
+ if @keepalive
545
+ info.node.socket_cache.revoke_by_value(info.sock)
546
+ end
516
547
  info.sock.close rescue nil
517
548
  rollback_write(info.chunk_id, update_retry: false)
518
549
  else
@@ -538,7 +569,142 @@ module Fluent::Plugin
538
569
  end
539
570
 
540
571
  class Node
541
- def initialize(sender, server, failure:)
572
+ class SocketCache
573
+ TimedSocket = Struct.new(:timeout, :sock, :ref)
574
+
575
+ def initialize(timeout, log)
576
+ @log = log
577
+ @timeout = timeout
578
+ @active_socks = {}
579
+ @inactive_socks = {}
580
+ @mutex = Mutex.new
581
+ end
582
+
583
+ def revoke(key = Thread.current.object_id)
584
+ @mutex.synchronize do
585
+ if @active_socks[key]
586
+ @inactive_socks[key] = @active_socks.delete(key)
587
+ @inactive_socks[key].ref = 0
588
+ end
589
+ end
590
+ end
591
+
592
+ def clear
593
+ @mutex.synchronize do
594
+ @inactive_socks.values.each do |s|
595
+ s.sock.close rescue nil
596
+ end
597
+ @inactive_socks.clear
598
+
599
+ @active_socks.values.each do |s|
600
+ s.sock.close rescue nil
601
+ end
602
+ @active_socks.clear
603
+ end
604
+ end
605
+
606
+ def purge_obsolete_socks
607
+ @mutex.synchronize do
608
+ @inactive_socks.keys.each do |k|
609
+ # 0 means sockets stored in this class received all acks
610
+ if @inactive_socks[k].ref <= 0
611
+ s = @inactive_socks.delete(k)
612
+ s.sock.close rescue nil
613
+ @log.debug("purged obsolete socket #{s.sock}")
614
+ end
615
+ end
616
+
617
+ @active_socks.keys.each do |k|
618
+ if expired?(k) && @active_socks[k].ref <= 0
619
+ @inactive_socks[k] = @active_socks.delete(k)
620
+ end
621
+ end
622
+ end
623
+ end
624
+
625
+ # We expect that `yield` returns a unique object in this class
626
+ def fetch_or(key = Thread.current.object_id)
627
+ @mutex.synchronize do
628
+ unless @active_socks[key]
629
+ @active_socks[key] = TimedSocket.new(timeout, yield, 1)
630
+ @log.debug("connect new socket #{@active_socks[key]}")
631
+ return @active_socks[key].sock
632
+ end
633
+
634
+ if expired?(key)
635
+ # Do not close this socket here in case of it will be used by other place (e.g. wait for receiving ack)
636
+ @inactive_socks[key] = @active_socks.delete(key)
637
+ @log.debug("connection #{@inactive_socks[key]} is expired. reconnecting...")
638
+ @active_socks[key] = TimedSocket.new(timeout, yield, 0)
639
+ end
640
+
641
+ @active_socks[key].ref += 1;
642
+ @active_socks[key].sock
643
+ end
644
+ end
645
+
646
+ def dec_ref(key = Thread.current.object_id)
647
+ @mutex.synchronize do
648
+ if @active_socks[key]
649
+ @active_socks[key].ref -= 1
650
+ elsif @inactive_socks[key]
651
+ @inactive_socks[key].ref -= 1
652
+ else
653
+ @log.warn("Not found key for dec_ref: #{key}")
654
+ end
655
+ end
656
+ end
657
+
658
+ # This method is expected to be called in class which doesn't call #inc_ref
659
+ def dec_ref_by_value(val)
660
+ @mutex.synchronize do
661
+ sock = @active_socks.detect { |_, v| v.sock == val }
662
+ if sock
663
+ key = sock.first
664
+ @active_socks[key].ref -= 1
665
+ return
666
+ end
667
+
668
+ sock = @inactive_socks.detect { |_, v| v.sock == val }
669
+ if sock
670
+ key = sock.first
671
+ @inactive_socks[key].ref -= 1
672
+ return
673
+ else
674
+ @log.warn("Not found key for dec_ref_by_value: #{key}")
675
+ end
676
+ end
677
+ end
678
+
679
+ # This method is expected to be called in class which doesn't call #fetch_or
680
+ def revoke_by_value(val)
681
+ @mutex.synchronize do
682
+ sock = @active_socks.detect { |_, v| v.sock == val }
683
+ if sock
684
+ key = sock.first
685
+ @inactive_socks[key] = @active_socks.delete(key)
686
+ @inactive_socks[key].ref = 0
687
+ else
688
+ @log.debug("Not found for revoke_by_value :#{val}")
689
+ end
690
+ end
691
+ end
692
+
693
+ private
694
+
695
+ def timeout
696
+ @timeout && Time.now + @timeout
697
+ end
698
+
699
+ # This method is thread unsafe
700
+ def expired?(key = Thread.current.object_id)
701
+ @active_socks[key].timeout ? @active_socks[key].timeout < Time.now : false
702
+ end
703
+ end
704
+
705
+ # @param keepalive [Bool]
706
+ # @param keepalive_timeout [Integer | nil]
707
+ def initialize(sender, server, failure:, keepalive: false, keepalive_timeout: nil)
542
708
  @sender = sender
543
709
  @log = sender.log
544
710
  @compress = sender.compress
@@ -571,6 +737,11 @@ module Fluent::Plugin
571
737
  @resolved_host = nil
572
738
  @resolved_time = 0
573
739
  @resolved_once = false
740
+
741
+ @keepalive = keepalive
742
+ if @keepalive
743
+ @socket_cache = SocketCache.new(keepalive_timeout, @log)
744
+ end
574
745
  end
575
746
 
576
747
  attr_accessor :usock
@@ -578,6 +749,7 @@ module Fluent::Plugin
578
749
  attr_reader :name, :host, :port, :weight, :standby, :state
579
750
  attr_reader :sockaddr # used by on_heartbeat
580
751
  attr_reader :failure, :available # for test
752
+ attr_reader :socket_cache # for ack
581
753
 
582
754
  RequestInfo = Struct.new(:state, :shared_key_nonce, :auth)
583
755
 
@@ -598,15 +770,12 @@ module Fluent::Plugin
598
770
  end
599
771
 
600
772
  def verify_connection
601
- sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
602
- begin
773
+ connect do |sock|
603
774
  ri = RequestInfo.new(@sender.security ? :helo : :established)
604
775
  if ri.state != :established
605
776
  establish_connection(sock, ri)
606
777
  raise if ri.state != :established
607
778
  end
608
- ensure
609
- sock.close
610
779
  end
611
780
  end
612
781
 
@@ -672,11 +841,16 @@ module Fluent::Plugin
672
841
  end
673
842
 
674
843
  def send_data(tag, chunk)
675
- sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
844
+ sock = connect
845
+
676
846
  begin
677
847
  send_data_actual(sock, tag, chunk)
678
848
  rescue
679
- sock.close rescue nil
849
+ if @keepalive
850
+ @socket_cache.revoke
851
+ else
852
+ sock.close rescue nil
853
+ end
680
854
  raise
681
855
  end
682
856
 
@@ -684,12 +858,27 @@ module Fluent::Plugin
684
858
  return sock # to read ACK from socket
685
859
  end
686
860
 
687
- sock.close_write rescue nil
688
- sock.close rescue nil
861
+ if @keepalive
862
+ @socket_cache.dec_ref
863
+ else
864
+ sock.close_write rescue nil
865
+ sock.close rescue nil
866
+ end
689
867
  heartbeat(false)
690
868
  nil
691
869
  end
692
870
 
871
+ def clear
872
+ @keepalive && @socket_cache.clear
873
+ end
874
+
875
+ def purge_obsolete_socks
876
+ unless @keepalive
877
+ raise "Don not call this method without keepalive option"
878
+ end
879
+ @socket_cache.purge_obsolete_socks
880
+ end
881
+
693
882
  # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
694
883
  def send_heartbeat
695
884
  begin
@@ -705,7 +894,7 @@ module Fluent::Plugin
705
894
 
706
895
  case @sender.heartbeat_type
707
896
  when :transport
708
- @sender.create_transfer_socket(dest_addr, port, @hostname) do |sock|
897
+ connect(dest_addr) do |sock|
709
898
  ## don't send any data to not cause a compatibility problem
710
899
  # sock.write FORWARD_TCP_HEARTBEAT_DATA
711
900
 
@@ -880,6 +1069,32 @@ module Fluent::Plugin
880
1069
  raise "BUG: unknown session state: #{ri.state}"
881
1070
  end
882
1071
  end
1072
+
1073
+ private
1074
+
1075
+ def connect(host = nil)
1076
+ sock = if @keepalive
1077
+ @socket_cache.fetch_or { @sender.create_transfer_socket(host || resolved_host, port, @hostname) }
1078
+ else
1079
+ @log.debug('connect new socket')
1080
+ @sender.create_transfer_socket(host || resolved_host, port, @hostname)
1081
+ end
1082
+
1083
+ if block_given?
1084
+ begin
1085
+ yield(sock)
1086
+ rescue
1087
+ @socket_cache.revoke(sock) if @keepalive
1088
+ raise
1089
+ else
1090
+ @socket_cache.dec_ref(sock) if @keepalive
1091
+ ensure
1092
+ sock.close unless @keepalive
1093
+ end
1094
+ else
1095
+ sock
1096
+ end
1097
+ end
883
1098
  end
884
1099
 
885
1100
  # Override Node to disable heartbeat