fluentd 0.14.11-x86-mingw32 → 0.14.12-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -5
  3. data/ChangeLog +54 -2
  4. data/example/in_dummy_blocks.conf +17 -0
  5. data/example/in_forward_tls.conf +14 -0
  6. data/example/in_forward_workers.conf +21 -0
  7. data/example/logevents.conf +25 -0
  8. data/example/out_forward_heartbeat_none.conf +16 -0
  9. data/example/out_forward_tls.conf +18 -0
  10. data/example/suppress_config_dump.conf +7 -0
  11. data/lib/fluent/agent.rb +3 -32
  12. data/lib/fluent/clock.rb +62 -0
  13. data/lib/fluent/command/fluentd.rb +12 -0
  14. data/lib/fluent/compat/input.rb +10 -1
  15. data/lib/fluent/compat/output.rb +40 -1
  16. data/lib/fluent/config/configure_proxy.rb +30 -7
  17. data/lib/fluent/config/section.rb +4 -0
  18. data/lib/fluent/config/types.rb +2 -2
  19. data/lib/fluent/configurable.rb +31 -5
  20. data/lib/fluent/engine.rb +61 -12
  21. data/lib/fluent/event_router.rb +6 -0
  22. data/lib/fluent/load.rb +0 -1
  23. data/lib/fluent/log.rb +118 -42
  24. data/lib/fluent/match.rb +37 -0
  25. data/lib/fluent/plugin.rb +25 -3
  26. data/lib/fluent/plugin/base.rb +4 -0
  27. data/lib/fluent/plugin/buf_file.rb +38 -14
  28. data/lib/fluent/plugin/buffer.rb +20 -20
  29. data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
  30. data/lib/fluent/plugin/compressable.rb +1 -0
  31. data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
  32. data/lib/fluent/plugin/formatter_csv.rb +4 -1
  33. data/lib/fluent/plugin/formatter_hash.rb +5 -1
  34. data/lib/fluent/plugin/formatter_json.rb +10 -0
  35. data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
  36. data/lib/fluent/plugin/in_dummy.rb +4 -0
  37. data/lib/fluent/plugin/in_exec.rb +4 -0
  38. data/lib/fluent/plugin/in_forward.rb +11 -3
  39. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  40. data/lib/fluent/plugin/in_http.rb +4 -0
  41. data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
  42. data/lib/fluent/plugin/in_object_space.rb +4 -1
  43. data/lib/fluent/plugin/in_syslog.rb +4 -0
  44. data/lib/fluent/plugin/in_tail.rb +193 -116
  45. data/lib/fluent/plugin/in_tcp.rb +5 -1
  46. data/lib/fluent/plugin/in_udp.rb +4 -0
  47. data/lib/fluent/plugin/input.rb +4 -0
  48. data/lib/fluent/plugin/out_copy.rb +4 -0
  49. data/lib/fluent/plugin/out_exec.rb +4 -0
  50. data/lib/fluent/plugin/out_exec_filter.rb +4 -0
  51. data/lib/fluent/plugin/out_file.rb +70 -30
  52. data/lib/fluent/plugin/out_forward.rb +132 -28
  53. data/lib/fluent/plugin/out_null.rb +10 -0
  54. data/lib/fluent/plugin/out_relabel.rb +4 -0
  55. data/lib/fluent/plugin/out_roundrobin.rb +4 -0
  56. data/lib/fluent/plugin/out_secondary_file.rb +5 -0
  57. data/lib/fluent/plugin/out_stdout.rb +5 -0
  58. data/lib/fluent/plugin/output.rb +18 -9
  59. data/lib/fluent/plugin/storage_local.rb +25 -2
  60. data/lib/fluent/plugin_helper/cert_option.rb +159 -0
  61. data/lib/fluent/plugin_helper/child_process.rb +6 -6
  62. data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
  63. data/lib/fluent/plugin_helper/event_loop.rb +29 -4
  64. data/lib/fluent/plugin_helper/inject.rb +14 -1
  65. data/lib/fluent/plugin_helper/server.rb +275 -31
  66. data/lib/fluent/plugin_helper/socket.rb +144 -4
  67. data/lib/fluent/plugin_helper/socket_option.rb +2 -17
  68. data/lib/fluent/plugin_helper/storage.rb +7 -1
  69. data/lib/fluent/plugin_helper/thread.rb +16 -4
  70. data/lib/fluent/registry.rb +26 -9
  71. data/lib/fluent/root_agent.rb +7 -3
  72. data/lib/fluent/supervisor.rb +37 -15
  73. data/lib/fluent/system_config.rb +37 -10
  74. data/lib/fluent/test.rb +2 -0
  75. data/lib/fluent/test/driver/base.rb +24 -26
  76. data/lib/fluent/test/helpers.rb +21 -0
  77. data/lib/fluent/version.rb +1 -1
  78. data/test/command/test_fluentd.rb +274 -4
  79. data/test/config/test_configurable.rb +154 -0
  80. data/test/config/test_configure_proxy.rb +180 -1
  81. data/test/config/test_system_config.rb +10 -0
  82. data/test/config/test_types.rb +1 -0
  83. data/test/plugin/test_base.rb +4 -0
  84. data/test/plugin/test_buf_file.rb +241 -9
  85. data/test/plugin/test_buffer.rb +11 -11
  86. data/test/plugin/test_buffer_file_chunk.rb +6 -6
  87. data/test/plugin/test_compressable.rb +3 -0
  88. data/test/plugin/test_filter.rb +4 -0
  89. data/test/plugin/test_filter_record_transformer.rb +20 -0
  90. data/test/plugin/test_formatter_csv.rb +9 -0
  91. data/test/plugin/test_formatter_hash.rb +35 -0
  92. data/test/plugin/test_formatter_json.rb +8 -0
  93. data/test/plugin/test_formatter_ltsv.rb +7 -0
  94. data/test/plugin/test_in_dummy.rb +7 -3
  95. data/test/plugin/test_in_monitor_agent.rb +43 -5
  96. data/test/plugin/test_in_tail.rb +97 -4
  97. data/test/plugin/test_input.rb +4 -0
  98. data/test/plugin/test_out_file.rb +46 -7
  99. data/test/plugin/test_out_forward.rb +59 -7
  100. data/test/plugin/test_output.rb +10 -4
  101. data/test/plugin/test_output_as_buffered.rb +37 -25
  102. data/test/plugin/test_output_as_buffered_compress.rb +1 -1
  103. data/test/plugin/test_output_as_buffered_retries.rb +6 -6
  104. data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
  105. data/test/plugin/test_storage_local.rb +40 -1
  106. data/test/plugin_helper/test_child_process.rb +29 -28
  107. data/test/plugin_helper/test_compat_parameters.rb +1 -1
  108. data/test/plugin_helper/test_inject.rb +27 -9
  109. data/test/plugin_helper/test_server.rb +822 -50
  110. data/test/plugin_helper/test_storage.rb +11 -0
  111. data/test/plugin_helper/test_timer.rb +1 -0
  112. data/test/test_clock.rb +164 -0
  113. data/test/test_log.rb +146 -15
  114. data/test/test_plugin.rb +251 -0
  115. data/test/test_supervisor.rb +65 -57
  116. data/test/test_test_drivers.rb +2 -2
  117. metadata +18 -7
  118. data/lib/fluent/process.rb +0 -504
  119. data/test/test_process.rb +0 -48
@@ -36,7 +36,7 @@ module Fluent
36
36
  "retry_limit" => "retry_max_times",
37
37
  "max_retry_wait" => "retry_max_interval",
38
38
  "buffer_chunk_limit" => "chunk_limit_size",
39
- "buffer_queue_limit" => "queue_length_limit",
39
+ "buffer_queue_limit" => "queue_limit_length",
40
40
  "buffer_queue_full_action" => "overflow_action",
41
41
  "flush_at_shutdown" => "flush_at_shutdown",
42
42
  }
@@ -16,6 +16,7 @@
16
16
 
17
17
  require 'cool.io'
18
18
  require 'fluent/plugin_helper/thread'
19
+ require 'fluent/clock'
19
20
 
20
21
  module Fluent
21
22
  module PluginHelper
@@ -31,7 +32,6 @@ module Fluent
31
32
 
32
33
  EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
33
34
  EVENT_LOOP_SHUTDOWN_TIMEOUT = 5
34
- EVENT_LOOP_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
35
35
 
36
36
  attr_reader :_event_loop # for tests
37
37
 
@@ -48,7 +48,14 @@ module Fluent
48
48
  end
49
49
 
50
50
  def event_loop_wait_until_stop
51
- sleep(0.1) while event_loop_running?
51
+ timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
52
+ sleep(0.1) while event_loop_running? && Fluent::Clock.now < timeout_at
53
+ if @_event_loop_running
54
+ puts "terminating event_loop forcedly"
55
+ caller.each{|bt| puts "\t#{bt}" }
56
+ @_event_loop.stop rescue nil
57
+ @_event_loop_running = true
58
+ end
52
59
  end
53
60
 
54
61
  def event_loop_running?
@@ -85,13 +92,31 @@ module Fluent
85
92
  @_event_loop_mutex.synchronize do
86
93
  @_event_loop_attached_watchers.reverse.each do |w|
87
94
  if w.attached?
95
+ begin
96
+ w.detach
97
+ rescue => e
98
+ log.warn "unexpected error while detaching event loop watcher", error: e
99
+ end
100
+ end
101
+ end
102
+ end
103
+
104
+ super
105
+ end
106
+
107
+ def after_shutdown
108
+ timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
109
+ @_event_loop_mutex.synchronize do
110
+ @_event_loop.watchers.reverse.each do |w|
111
+ begin
88
112
  w.detach
113
+ rescue => e
114
+ log.warn "unexpected error while detaching event loop watcher", error: e
89
115
  end
90
116
  end
91
117
  end
92
- timeout_at = Process.clock_gettime(EVENT_LOOP_CLOCK_ID) + EVENT_LOOP_SHUTDOWN_TIMEOUT
93
118
  while @_event_loop_running
94
- if Process.clock_gettime(EVENT_LOOP_CLOCK_ID) >= timeout_at
119
+ if Fluent::Clock.now >= timeout_at
95
120
  log.warn "event loop does NOT exit until hard timeout."
96
121
  raise "event loop does NOT exit until hard timeout." if @under_plugin_development
97
122
  break
@@ -29,6 +29,9 @@ module Fluent
29
29
  if @_inject_hostname_key
30
30
  r[@_inject_hostname_key] = @_inject_hostname
31
31
  end
32
+ if @_inject_worker_id_key
33
+ r[@_inject_worker_id_key] = @_inject_worker_id
34
+ end
32
35
  if @_inject_tag_key
33
36
  r[@_inject_tag_key] = tag
34
37
  end
@@ -48,6 +51,9 @@ module Fluent
48
51
  if @_inject_hostname_key
49
52
  r[@_inject_hostname_key] = @_inject_hostname
50
53
  end
54
+ if @_inject_worker_id_key
55
+ r[@_inject_worker_id_key] = @_inject_worker_id
56
+ end
51
57
  if @_inject_tag_key
52
58
  r[@_inject_tag_key] = tag
53
59
  end
@@ -65,6 +71,7 @@ module Fluent
65
71
  config_section :inject, required: false, multi: false, param_name: :inject_config do
66
72
  config_param :hostname_key, :string, default: nil
67
73
  config_param :hostname, :string, default: nil
74
+ config_param :worker_id_key, :string, default: nil
68
75
  config_param :tag_key, :string, default: nil
69
76
  config_param :time_key, :string, default: nil
70
77
 
@@ -86,6 +93,8 @@ module Fluent
86
93
  @_inject_enabled = false
87
94
  @_inject_hostname_key = nil
88
95
  @_inject_hostname = nil
96
+ @_inject_worker_id_key = nil
97
+ @_inject_worker_id = nil
89
98
  @_inject_tag_key = nil
90
99
  @_inject_time_key = nil
91
100
  @_inject_time_formatter = nil
@@ -114,6 +123,10 @@ module Fluent
114
123
  log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
115
124
  end
116
125
  end
126
+ @_inject_worker_id_key = @inject_config.worker_id_key
127
+ if @_inject_worker_id_key
128
+ @_inject_worker_id = fluentd_worker_id # get id here, because #with_worker_config method may be used only for #configure in tests
129
+ end
117
130
  @_inject_tag_key = @inject_config.tag_key
118
131
  @_inject_time_key = @inject_config.time_key
119
132
  if @_inject_time_key
@@ -130,7 +143,7 @@ module Fluent
130
143
  end
131
144
  end
132
145
 
133
- @_inject_enabled = @_inject_hostname_key || @_inject_tag_key || @_inject_time_key
146
+ @_inject_enabled = @_inject_hostname_key || @_inject_worker_id_key || @_inject_tag_key || @_inject_time_key
134
147
  end
135
148
  end
136
149
  end
@@ -21,18 +21,22 @@ require 'cool.io'
21
21
  require 'socket'
22
22
  require 'ipaddr'
23
23
  require 'fcntl'
24
+ require 'openssl'
24
25
 
25
26
  require_relative 'socket_option'
27
+ require_relative 'cert_option'
26
28
 
27
29
  module Fluent
28
30
  module PluginHelper
29
31
  module Server
30
32
  include Fluent::PluginHelper::EventLoop
31
33
  include Fluent::PluginHelper::SocketOption
34
+ include Fluent::PluginHelper::CertOption
32
35
 
33
36
  # This plugin helper doesn't support these things for now:
34
- # * SSL/TLS (TBD)
35
37
  # * TCP/TLS keepalive
38
+ # * TLS session cache/tickets
39
+ # * unix domain sockets
36
40
 
37
41
  # stop : [-]
38
42
  # shutdown : detach server event handler from event loop (event_loop)
@@ -63,12 +67,16 @@ module Fluent
63
67
  # conn.close
64
68
  # end
65
69
  # end
66
- def server_create_connection(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, backlog: nil, **socket_options, &block)
70
+ def server_create_connection(title, port, proto: nil, bind: '0.0.0.0', shared: true, backlog: nil, tls_options: nil, **socket_options, &block)
71
+ proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
72
+
67
73
  raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
68
74
  raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
69
75
  raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
70
76
  raise ArgumentError, "BUG: cannot create connection for UDP" unless CONNECTION_PROTOCOLS.include?(proto)
71
77
 
78
+ raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
79
+
72
80
  raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
73
81
  raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
74
82
 
@@ -83,11 +91,14 @@ module Fluent
83
91
  when :tcp
84
92
  server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
85
93
  when :tls
86
- raise ArgumentError, "BUG: certopts (certificate options) not specified for TLS" unless certopts
87
- # server_certopts_validate!(certopts)
88
- # sock = server_create_tls_socket(shared, bind, port)
89
- # server = nil # ...
90
- raise "not implemented yet"
94
+ transport_config = if tls_options
95
+ server_create_transport_section_object(tls_options)
96
+ elsif @transport_config && @transport_config.protocol == :tls
97
+ @transport_config
98
+ else
99
+ raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
100
+ end
101
+ server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter, &block)
91
102
  when :unix
92
103
  raise "not implemented yet"
93
104
  else
@@ -108,12 +119,15 @@ module Fluent
108
119
  # sock.remote_port
109
120
  # # ...
110
121
  # end
111
- def server_create(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
122
+ def server_create(title, port, proto: nil, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, tls_options: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
123
+ proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
124
+
112
125
  raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
113
126
  raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
114
127
  raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
115
128
 
116
129
  raise ArgumentError, "BUG: socket option is available only for udp" if socket && proto != :udp
130
+ raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
117
131
 
118
132
  raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
119
133
  raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
@@ -141,7 +155,16 @@ module Fluent
141
155
  conn.data(&callback)
142
156
  end
143
157
  when :tls
144
- raise "not implemented yet"
158
+ transport_config = if tls_options
159
+ server_create_transport_section_object(tls_options)
160
+ elsif @transport_config && @transport_config.protocol == :tls
161
+ @transport_config
162
+ else
163
+ raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
164
+ end
165
+ server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter) do |conn|
166
+ conn.data(&callback)
167
+ end
145
168
  when :udp
146
169
  raise ArgumentError, "BUG: max_bytes must be specified for UDP" unless max_bytes
147
170
  if socket
@@ -198,6 +221,80 @@ module Fluent
198
221
  server
199
222
  end
200
223
 
224
+ def server_create_for_tls_connection(shared, bind, port, conf, backlog, socket_option_setter, &block)
225
+ context = cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)
226
+ sock = server_create_tcp_socket(shared, bind, port)
227
+ socket_option_setter.call(sock)
228
+ close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
229
+ server = Coolio::TCPServer.new(sock, nil, EventHandler::TLSServer, context, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
230
+ @_server_mutex.synchronize do
231
+ @_server_connections << conn
232
+ end
233
+ end
234
+ server.listen(backlog) if backlog
235
+ server
236
+ end
237
+
238
+ SERVER_TRANSPORT_PARAMS = [
239
+ :protocol, :version, :ciphers, :insecure,
240
+ :cert_path, :private_key_path, :private_key_passphrase,
241
+ :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
242
+ :generate_private_key_length,
243
+ :generate_cert_country, :generate_cert_state, :generate_cert_state,
244
+ :generate_cert_locality, :generate_cert_common_name,
245
+ :generate_cert_expiration, :generate_cert_digest,
246
+ ]
247
+
248
+ def server_create_transport_section_object(opts)
249
+ transport_section = configured_section_create(:transport)
250
+ SERVER_TRANSPORT_PARAMS.each do |param|
251
+ if opts.has_key?(param)
252
+ transport_section[param] = opts[param]
253
+ end
254
+ end
255
+ transport_section
256
+ end
257
+
258
+ module ServerTransportParams
259
+ TLS_DEFAULT_VERSION = :'TLSv1_2'
260
+ TLS_SUPPORTED_VERSIONS = [:'TLSv1_1', :'TLSv1_2']
261
+ ### follow httpclient configuration by nahi
262
+ # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
263
+ CIPHERS_DEFAULT = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
264
+
265
+ include Fluent::Configurable
266
+ config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
267
+ config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
268
+ config_param :version, :enum, list: TLS_SUPPORTED_VERSIONS, default: TLS_DEFAULT_VERSION
269
+
270
+ config_param :ciphers, :string, default: CIPHERS_DEFAULT
271
+ config_param :insecure, :bool, default: false
272
+
273
+ # Cert signed by public CA
274
+ config_param :cert_path, :string, default: nil
275
+ config_param :private_key_path, :string, default: nil
276
+ config_param :private_key_passphrase, :string, default: nil, secret: true
277
+
278
+ # Cert generated and signed by private CA Certificate
279
+ config_param :ca_cert_path, :string, default: nil
280
+ config_param :ca_private_key_path, :string, default: nil
281
+ config_param :ca_private_key_passphrase, :string, default: nil, secret: true
282
+
283
+ # Options for generating certs by private CA certs or self-signed
284
+ config_param :generate_private_key_length, :integer, default: 2048
285
+ config_param :generate_cert_country, :string, default: 'US'
286
+ config_param :generate_cert_state, :string, default: 'CA'
287
+ config_param :generate_cert_locality, :string, default: 'Mountain View'
288
+ config_param :generate_cert_common_name, :string, default: nil
289
+ config_param :generate_cert_expiration, :time, default: 10 * 365 * 86400 # 10years later
290
+ config_param :generate_cert_digest, :enum, list: [:sha1, :sha256, :sha384, :sha512], default: :sha256
291
+ end
292
+ end
293
+
294
+ def self.included(mod)
295
+ mod.include ServerTransportParams
296
+ end
297
+
201
298
  def initialize
202
299
  super
203
300
  @_servers = []
@@ -205,28 +302,33 @@ module Fluent
205
302
  @_server_mutex = Mutex.new
206
303
  end
207
304
 
208
- def shutdown
209
- @_server_connections.each do |conn|
210
- conn.close rescue nil
305
+ def configure(conf)
306
+ super
307
+
308
+ if @transport_config
309
+ if @transport_config.protocol == :tls
310
+ cert_option_server_validate!(@transport_config)
311
+ end
211
312
  end
313
+ end
314
+
315
+ def stop
212
316
  @_server_mutex.synchronize do
213
317
  @_servers.each do |si|
214
318
  si.server.detach if si.server.attached?
319
+ # to refuse more connections: (connected sockets are still alive here)
320
+ si.server.close rescue nil
215
321
  end
216
322
  end
217
323
 
218
324
  super
219
325
  end
220
326
 
221
- def close
327
+ def shutdown
222
328
  @_server_connections.each do |conn|
223
329
  conn.close rescue nil
224
330
  end
225
- @_server_mutex.synchronize do
226
- @_servers.each do |si|
227
- si.server.close rescue nil
228
- end
229
- end
331
+
230
332
  super
231
333
  end
232
334
 
@@ -235,10 +337,6 @@ module Fluent
235
337
  super
236
338
  end
237
339
 
238
- def server_certopts_validate!(certopts)
239
- raise "not implemented yet"
240
- end
241
-
242
340
  def server_socket_manager_client
243
341
  socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
244
342
  if Fluent.windows?
@@ -272,28 +370,25 @@ module Fluent
272
370
  sock
273
371
  end
274
372
 
275
- def server_create_tls_socket(shared, bind, port)
276
- raise "not implemented yet"
277
- end
278
-
279
373
  class CallbackSocket
280
374
  def initialize(server_type, sock, enabled_events = [], close_socket: true)
281
375
  @server_type = server_type
282
376
  @sock = sock
377
+ @peeraddr = nil
283
378
  @enabled_events = enabled_events
284
379
  @close_socket = close_socket
285
380
  end
286
381
 
287
382
  def remote_addr
288
- @sock.peeraddr[3]
383
+ @peeraddr[3]
289
384
  end
290
385
 
291
386
  def remote_host
292
- @sock.peeraddr[2]
387
+ @peeraddr[2]
293
388
  end
294
389
 
295
390
  def remote_port
296
- @sock.peeraddr[1]
391
+ @peeraddr[1]
297
392
  end
298
393
 
299
394
  def send(data, flags = 0)
@@ -332,6 +427,18 @@ module Fluent
332
427
  class TCPCallbackSocket < CallbackSocket
333
428
  def initialize(sock)
334
429
  super("tcp", sock, [:data, :write_complete, :close])
430
+ @peeraddr = @sock.peeraddr
431
+ end
432
+
433
+ def write(data)
434
+ @sock.write(data)
435
+ end
436
+ end
437
+
438
+ class TLSCallbackSocket < CallbackSocket
439
+ def initialize(sock)
440
+ super("tls", sock, [:data, :write_complete, :close])
441
+ @peeraddr = @sock.to_io.peeraddr
335
442
  end
336
443
 
337
444
  def write(data)
@@ -435,6 +542,10 @@ module Fluent
435
542
  @mutex = Mutex.new # to serialize #write and #close
436
543
  end
437
544
 
545
+ def to_io
546
+ @_handler_socket
547
+ end
548
+
438
549
  def data(&callback)
439
550
  raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
440
551
  @data_callback = callback
@@ -464,7 +575,140 @@ module Fluent
464
575
  def on_read_without_connection(data)
465
576
  @data_callback.call(data)
466
577
  rescue => e
467
- @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
578
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
579
+ @log.error_backtrace
580
+ close(true) rescue nil
581
+ raise if @under_plugin_development
582
+ end
583
+
584
+ def on_read_with_connection(data)
585
+ @data_callback.call(data, @callback_connection)
586
+ rescue => e
587
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
588
+ @log.error_backtrace
589
+ close(true) rescue nil
590
+ raise if @under_plugin_development
591
+ end
592
+
593
+ def close
594
+ @mutex.synchronize do
595
+ return if @closing
596
+ @closing = true
597
+ @close_callback.call(self)
598
+ super
599
+ end
600
+ end
601
+ end
602
+
603
+ class TLSServer < Coolio::Socket
604
+ # It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.
605
+ def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
606
+ raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
607
+
608
+ socket_option_setter.call(sock)
609
+ @_handler_socket = OpenSSL::SSL::SSLSocket.new(sock, context)
610
+ @_handler_socket.sync_close = true
611
+ @_handler_write_buffer = ''.force_encoding('ascii-8bit')
612
+ @_handler_accepted = false
613
+ super(@_handler_socket)
614
+
615
+ @log = log
616
+ @under_plugin_development = under_plugin_development
617
+
618
+ @connect_callback = connect_callback
619
+ @data_callback = nil
620
+ @close_callback = close_callback
621
+
622
+ @callback_connection = nil
623
+ @closing = false
624
+
625
+ @mutex = Mutex.new # to serialize #write and #close
626
+ end
627
+
628
+ def to_io
629
+ @_handler_socket.to_io
630
+ end
631
+
632
+ def data(&callback)
633
+ raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
634
+ @data_callback = callback
635
+ on_read_impl = case callback.arity
636
+ when 1 then :on_read_without_connection
637
+ when 2 then :on_read_with_connection
638
+ else
639
+ raise "BUG: callback block must have 1 or 2 arguments"
640
+ end
641
+ self.define_singleton_method(:on_read, method(on_read_impl))
642
+ end
643
+
644
+ def write(data)
645
+ @mutex.synchronize do
646
+ @_handler_write_buffer << data
647
+ schedule_write
648
+ data.bytesize
649
+ end
650
+ end
651
+
652
+ def try_tls_accept
653
+ return true if @_handler_accepted
654
+
655
+ begin
656
+ @_handler_socket.accept_nonblock # this method call actually try to do handshake via TLS
657
+ @_handler_accepted = true
658
+
659
+ @callback_connection = TLSCallbackSocket.new(self)
660
+ @connect_callback.call(@callback_connection)
661
+ unless @data_callback
662
+ raise "connection callback must call #data to set data callback"
663
+ end
664
+ return true
665
+
666
+ rescue IO::WaitReadable, IO::WaitWritable
667
+ # retry accept_nonblock: there aren't enough data in underlying socket buffer
668
+ rescue OpenSSL::SSL::SSLError => e
669
+ @log.trace "unexpected error before accepting TLS connection", error: e
670
+ close rescue nil
671
+ end
672
+
673
+ false
674
+ end
675
+
676
+ def on_connect
677
+ try_tls_accept
678
+ end
679
+
680
+ def on_readable
681
+ if try_tls_accept
682
+ super
683
+ end
684
+ rescue IO::WaitReadable, IO::WaitWritable
685
+ # ignore and return with doing nothing
686
+ end
687
+
688
+ def on_writable
689
+ begin
690
+ @mutex.synchronize do
691
+ written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
692
+ @_handler_write_buffer.slice!(0, written_bytes)
693
+ super
694
+ end
695
+ rescue IO::WaitWritable, IO::WaitReadable
696
+ return
697
+ rescue Errno::EINTR
698
+ return
699
+ rescue SystemCallError, IOError, SocketError
700
+ # SystemCallError catches Errno::EPIPE & Errno::ECONNRESET amongst others.
701
+ close rescue nil
702
+ return
703
+ rescue OpenSSL::SSL::SSLError => e
704
+ @log.debug "unexpected SSLError while writing data into socket connected via TLS", error: e
705
+ end
706
+ end
707
+
708
+ def on_read_without_connection(data)
709
+ @data_callback.call(data)
710
+ rescue => e
711
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
468
712
  @log.error_backtrace
469
713
  close(true) rescue nil
470
714
  raise if @under_plugin_development
@@ -473,7 +717,7 @@ module Fluent
473
717
  def on_read_with_connection(data)
474
718
  @data_callback.call(data, @callback_connection)
475
719
  rescue => e
476
- @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
720
+ @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
477
721
  @log.error_backtrace
478
722
  close(true) rescue nil
479
723
  raise if @under_plugin_development