fluentd 0.14.11 → 0.14.12

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -5
  3. data/ChangeLog +54 -2
  4. data/example/in_dummy_blocks.conf +17 -0
  5. data/example/in_forward_tls.conf +14 -0
  6. data/example/in_forward_workers.conf +21 -0
  7. data/example/logevents.conf +25 -0
  8. data/example/out_forward_heartbeat_none.conf +16 -0
  9. data/example/out_forward_tls.conf +18 -0
  10. data/example/suppress_config_dump.conf +7 -0
  11. data/lib/fluent/agent.rb +3 -32
  12. data/lib/fluent/clock.rb +62 -0
  13. data/lib/fluent/command/fluentd.rb +12 -0
  14. data/lib/fluent/compat/input.rb +10 -1
  15. data/lib/fluent/compat/output.rb +40 -1
  16. data/lib/fluent/config/configure_proxy.rb +30 -7
  17. data/lib/fluent/config/section.rb +4 -0
  18. data/lib/fluent/config/types.rb +2 -2
  19. data/lib/fluent/configurable.rb +31 -5
  20. data/lib/fluent/engine.rb +61 -12
  21. data/lib/fluent/event_router.rb +6 -0
  22. data/lib/fluent/load.rb +0 -1
  23. data/lib/fluent/log.rb +118 -42
  24. data/lib/fluent/match.rb +37 -0
  25. data/lib/fluent/plugin.rb +25 -3
  26. data/lib/fluent/plugin/base.rb +4 -0
  27. data/lib/fluent/plugin/buf_file.rb +38 -14
  28. data/lib/fluent/plugin/buffer.rb +20 -20
  29. data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
  30. data/lib/fluent/plugin/compressable.rb +1 -0
  31. data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
  32. data/lib/fluent/plugin/formatter_csv.rb +4 -1
  33. data/lib/fluent/plugin/formatter_hash.rb +5 -1
  34. data/lib/fluent/plugin/formatter_json.rb +10 -0
  35. data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
  36. data/lib/fluent/plugin/in_dummy.rb +4 -0
  37. data/lib/fluent/plugin/in_exec.rb +4 -0
  38. data/lib/fluent/plugin/in_forward.rb +11 -3
  39. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  40. data/lib/fluent/plugin/in_http.rb +4 -0
  41. data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
  42. data/lib/fluent/plugin/in_object_space.rb +4 -1
  43. data/lib/fluent/plugin/in_syslog.rb +4 -0
  44. data/lib/fluent/plugin/in_tail.rb +193 -116
  45. data/lib/fluent/plugin/in_tcp.rb +5 -1
  46. data/lib/fluent/plugin/in_udp.rb +4 -0
  47. data/lib/fluent/plugin/input.rb +4 -0
  48. data/lib/fluent/plugin/out_copy.rb +4 -0
  49. data/lib/fluent/plugin/out_exec.rb +4 -0
  50. data/lib/fluent/plugin/out_exec_filter.rb +4 -0
  51. data/lib/fluent/plugin/out_file.rb +70 -30
  52. data/lib/fluent/plugin/out_forward.rb +132 -28
  53. data/lib/fluent/plugin/out_null.rb +10 -0
  54. data/lib/fluent/plugin/out_relabel.rb +4 -0
  55. data/lib/fluent/plugin/out_roundrobin.rb +4 -0
  56. data/lib/fluent/plugin/out_secondary_file.rb +5 -0
  57. data/lib/fluent/plugin/out_stdout.rb +5 -0
  58. data/lib/fluent/plugin/output.rb +18 -9
  59. data/lib/fluent/plugin/storage_local.rb +25 -2
  60. data/lib/fluent/plugin_helper/cert_option.rb +159 -0
  61. data/lib/fluent/plugin_helper/child_process.rb +6 -6
  62. data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
  63. data/lib/fluent/plugin_helper/event_loop.rb +29 -4
  64. data/lib/fluent/plugin_helper/inject.rb +14 -1
  65. data/lib/fluent/plugin_helper/server.rb +275 -31
  66. data/lib/fluent/plugin_helper/socket.rb +144 -4
  67. data/lib/fluent/plugin_helper/socket_option.rb +2 -17
  68. data/lib/fluent/plugin_helper/storage.rb +7 -1
  69. data/lib/fluent/plugin_helper/thread.rb +16 -4
  70. data/lib/fluent/registry.rb +26 -9
  71. data/lib/fluent/root_agent.rb +7 -3
  72. data/lib/fluent/supervisor.rb +37 -15
  73. data/lib/fluent/system_config.rb +37 -10
  74. data/lib/fluent/test.rb +2 -0
  75. data/lib/fluent/test/driver/base.rb +24 -26
  76. data/lib/fluent/test/helpers.rb +21 -0
  77. data/lib/fluent/version.rb +1 -1
  78. data/test/command/test_fluentd.rb +274 -4
  79. data/test/config/test_configurable.rb +154 -0
  80. data/test/config/test_configure_proxy.rb +180 -1
  81. data/test/config/test_system_config.rb +10 -0
  82. data/test/config/test_types.rb +1 -0
  83. data/test/plugin/test_base.rb +4 -0
  84. data/test/plugin/test_buf_file.rb +241 -9
  85. data/test/plugin/test_buffer.rb +11 -11
  86. data/test/plugin/test_buffer_file_chunk.rb +6 -6
  87. data/test/plugin/test_compressable.rb +3 -0
  88. data/test/plugin/test_filter.rb +4 -0
  89. data/test/plugin/test_filter_record_transformer.rb +20 -0
  90. data/test/plugin/test_formatter_csv.rb +9 -0
  91. data/test/plugin/test_formatter_hash.rb +35 -0
  92. data/test/plugin/test_formatter_json.rb +8 -0
  93. data/test/plugin/test_formatter_ltsv.rb +7 -0
  94. data/test/plugin/test_in_dummy.rb +7 -3
  95. data/test/plugin/test_in_monitor_agent.rb +43 -5
  96. data/test/plugin/test_in_tail.rb +97 -4
  97. data/test/plugin/test_input.rb +4 -0
  98. data/test/plugin/test_out_file.rb +46 -7
  99. data/test/plugin/test_out_forward.rb +59 -7
  100. data/test/plugin/test_output.rb +10 -4
  101. data/test/plugin/test_output_as_buffered.rb +37 -25
  102. data/test/plugin/test_output_as_buffered_compress.rb +1 -1
  103. data/test/plugin/test_output_as_buffered_retries.rb +6 -6
  104. data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
  105. data/test/plugin/test_storage_local.rb +40 -1
  106. data/test/plugin_helper/test_child_process.rb +29 -28
  107. data/test/plugin_helper/test_compat_parameters.rb +1 -1
  108. data/test/plugin_helper/test_inject.rb +27 -9
  109. data/test/plugin_helper/test_server.rb +822 -50
  110. data/test/plugin_helper/test_storage.rb +11 -0
  111. data/test/plugin_helper/test_timer.rb +1 -0
  112. data/test/test_clock.rb +164 -0
  113. data/test/test_log.rb +146 -15
  114. data/test/test_plugin.rb +251 -0
  115. data/test/test_supervisor.rb +65 -57
  116. data/test/test_test_drivers.rb +2 -2
  117. metadata +18 -7
  118. data/lib/fluent/process.rb +0 -504
  119. data/test/test_process.rb +0 -48
@@ -84,7 +84,7 @@ class LocalStorageTest < Test::Unit::TestCase
84
84
  end
85
85
  end
86
86
 
87
- sub_test_case 'configured with path' do
87
+ sub_test_case 'configured with file path' do
88
88
  test 'works as storage which stores data on disk' do
89
89
  storage_path = File.join(TMP_DIR, 'my_store.json')
90
90
  conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path})])
@@ -127,6 +127,15 @@ class LocalStorageTest < Test::Unit::TestCase
127
127
  assert_equal '2', @p.get('key1')
128
128
  assert_equal 4, @p.get('key2')
129
129
  end
130
+
131
+ test 'raise configuration error if a file specified with multi worker configuration' do
132
+ storage_path = File.join(TMP_DIR, 'my_store.json')
133
+ conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path})])
134
+ @d.system_config_override('workers' => 3)
135
+ assert_raise Fluent::ConfigError.new("Plugin 'local' does not support multi workers configuration (Fluent::Plugin::LocalStorage)") do
136
+ @d.configure(conf)
137
+ end
138
+ end
130
139
  end
131
140
 
132
141
  sub_test_case 'configured with root-dir and plugin id' do
@@ -177,6 +186,36 @@ class LocalStorageTest < Test::Unit::TestCase
177
186
  assert_equal '2', @p.get('key1')
178
187
  assert_equal 4, @p.get('key2')
179
188
  end
189
+
190
+ test 'works with customized path by specified usage' do
191
+ root_dir = File.join(TMP_DIR, 'root')
192
+ expected_storage_path = File.join(root_dir, 'worker0', 'local_storage_test', 'storage.usage.json')
193
+ conf = config_element('ROOT', 'usage', {'@id' => 'local_storage_test'})
194
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do
195
+ @d.configure(conf)
196
+ end
197
+ @d.start
198
+ @p = @d.storage_create(usage: 'usage', type: 'local')
199
+
200
+ assert_equal expected_storage_path, @p.path
201
+ assert @p.store.empty?
202
+ end
203
+ end
204
+
205
+ sub_test_case 'configured with root-dir and plugin id, and multi workers' do
206
+ test 'works as storage which stores data under root dir, also in workers' do
207
+ root_dir = File.join(TMP_DIR, 'root')
208
+ expected_storage_path = File.join(root_dir, 'worker1', 'local_storage_test', 'storage.json')
209
+ conf = config_element('ROOT', '', {'@id' => 'local_storage_test'})
210
+ with_worker_config(root_dir: root_dir, workers: 2, worker_id: 1) do
211
+ @d.configure(conf)
212
+ end
213
+ @d.start
214
+ @p = @d.storage_create()
215
+
216
+ assert_equal expected_storage_path, @p.path
217
+ assert @p.store.empty?
218
+ end
180
219
  end
181
220
 
182
221
  sub_test_case 'persistent specified' do
@@ -630,17 +630,15 @@ class ChildProcessTest < Test::Unit::TestCase
630
630
 
631
631
  str = nil
632
632
 
633
- Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
634
- pid = nil
635
- @d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
636
- pid = @d.instance_eval{ child_process_id }
637
- str = readio.read.chomp
638
- block_exits = true
639
- end
640
- sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) # to get exit status
641
- sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits
642
- sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
633
+ pid = nil
634
+ @d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
635
+ pid = @d.instance_eval{ child_process_id }
636
+ str = readio.read.chomp
637
+ block_exits = true
643
638
  end
639
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) } # to get exit status
640
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits }
641
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called }
644
642
 
645
643
  assert callback_called
646
644
  assert exit_status
@@ -658,32 +656,35 @@ class ChildProcessTest < Test::Unit::TestCase
658
656
  block_exits = false
659
657
  callback_called = false
660
658
  exit_status = nil
661
- args = ['-e', 'sleep ARGV[0].to_i; puts "yay"; File.unlink ARGV[1]', '25', @temp_path]
659
+ args = ['-e', 'sleep ARGV[0].to_i; puts "yay"; File.unlink ARGV[1]', '100', @temp_path]
662
660
  cb = ->(status){ exit_status = status; callback_called = true }
663
661
 
664
662
  str = nil
665
663
 
666
- Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
667
- pid = nil
668
- @d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
669
- pid = @d.instance_eval{ child_process_id }
670
- Process.kill(:QUIT, pid)
671
- Process.kill(:QUIT, pid) rescue nil # once more to kill certainly
672
- str = readio.read.chomp rescue nil # empty string before EOF
673
- block_exits = true
674
- end
675
- sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) # to get exit status
676
- sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits
677
- sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
678
- end
664
+ pid = nil
665
+ @d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
666
+ pid = @d.instance_eval{ child_process_id }
667
+ sleep 10 # to run child process correctly
668
+ Process.kill(:QUIT, pid)
669
+ sleep 1
670
+ Process.kill(:QUIT, pid) rescue nil # once more to send kill
671
+ sleep 1
672
+ Process.kill(:QUIT, pid) rescue nil # just like sync
673
+ str = readio.read.chomp rescue nil # empty string before EOF
674
+ block_exits = true
675
+ end
676
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) } # to get exit status
677
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits }
678
+ waiting(TEST_DEADLOCK_TIMEOUT){ sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called }
679
679
 
680
680
  assert callback_called
681
681
  assert exit_status
682
682
 
683
- assert_equal [nil, 3], [exit_status.exitstatus, exit_status.termsig] # SIGQUIT
684
-
685
- assert File.exist?(@temp_path)
686
- assert_equal "", str
683
+ # This test sometimes fails on TravisCI
684
+ # with [nil, 11] # SIGSEGV
685
+ # or with [1, nil] # ???
686
+ assert_equal [nil, 3, true, ""], [exit_status.exitstatus, exit_status.termsig, File.exist?(@temp_path), str] # SIGQUIT
687
+ # SIGSEGV looks a kind of BUG of ruby...
687
688
  end
688
689
 
689
690
  test 'calls on_exit_callback for each process exits for interval call using on_exit_callback' do
@@ -109,7 +109,7 @@ class CompatParameterTest < Test::Unit::TestCase
109
109
  assert @i.buffer_config.flush_at_shutdown
110
110
 
111
111
  assert_equal 8*1024*1024, @i.buffer.chunk_limit_size
112
- assert_equal 1024, @i.buffer.queue_length_limit
112
+ assert_equal 1024, @i.buffer.queue_limit_length
113
113
  end
114
114
  end
115
115
 
@@ -59,6 +59,8 @@ class InjectHelperTest < Test::Unit::TestCase
59
59
  @d.start
60
60
  assert_nil @d.instance_eval{ @_inject_hostname_key }
61
61
  assert_nil @d.instance_eval{ @_inject_hostname }
62
+ assert_nil @d.instance_eval{ @_inject_worker_id_key }
63
+ assert_nil @d.instance_eval{ @_inject_worker_id }
62
64
  assert_nil @d.instance_eval{ @_inject_tag_key }
63
65
  assert_nil @d.instance_eval{ @_inject_time_key }
64
66
  assert_nil @d.instance_eval{ @_inject_time_formatter }
@@ -85,18 +87,23 @@ class InjectHelperTest < Test::Unit::TestCase
85
87
  end
86
88
 
87
89
  test 'can be configured as specified' do
88
- @d.configure(config_inject_section(
89
- "hostname_key" => "hostname",
90
- "hostname" => "myhost.local",
91
- "tag_key" => "tag",
92
- "time_key" => "time",
93
- "time_type" => "string",
94
- "time_format" => "%Y-%m-%d %H:%M:%S.%N",
95
- "timezone" => "-0700",
96
- ))
90
+ with_worker_config(workers: 1, worker_id: 0) do
91
+ @d.configure(config_inject_section(
92
+ "hostname_key" => "hostname",
93
+ "hostname" => "myhost.local",
94
+ "worker_id_key" => "worker_id",
95
+ "tag_key" => "tag",
96
+ "time_key" => "time",
97
+ "time_type" => "string",
98
+ "time_format" => "%Y-%m-%d %H:%M:%S.%N",
99
+ "timezone" => "-0700",
100
+ ))
101
+ end
97
102
 
98
103
  assert_equal "hostname", @d.instance_eval{ @_inject_hostname_key }
99
104
  assert_equal "myhost.local", @d.instance_eval{ @_inject_hostname }
105
+ assert_equal "worker_id", @d.instance_eval{ @_inject_worker_id_key }
106
+ assert_equal 0, @d.instance_eval{ @_inject_worker_id }
100
107
  assert_equal "tag", @d.instance_eval{ @_inject_tag_key }
101
108
  assert_equal "time", @d.instance_eval{ @_inject_time_key }
102
109
  assert_equal :string, @d.instance_eval{ @inject_config.time_type }
@@ -136,6 +143,17 @@ class InjectHelperTest < Test::Unit::TestCase
136
143
  assert_equal record.merge({"host" => "myhost.yay.local"}), @d.inject_values_to_record('tag', time, record)
137
144
  end
138
145
 
146
+ test 'injects worker id' do
147
+ with_worker_config(workers: 3, worker_id: 2) do
148
+ @d.configure(config_inject_section("worker_id_key" => "workerid"))
149
+ end
150
+ @d.start
151
+
152
+ time = event_time()
153
+ record = {"key1" => "value1", "key2" => 2}
154
+ assert_equal record.merge({"workerid" => 2}), @d.inject_values_to_record('tag', time, record)
155
+ end
156
+
139
157
  test 'injects tag into specified key' do
140
158
  @d.configure(config_inject_section("tag_key" => "mytag"))
141
159
  @d.start
@@ -1,5 +1,6 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin_helper/server'
3
+ require 'fluent/plugin_helper/cert_option' # to create certs for tests
3
4
  require 'fluent/plugin/base'
4
5
  require 'timeout'
5
6
 
@@ -11,6 +12,8 @@ class ServerPluginHelperTest < Test::Unit::TestCase
11
12
  helpers :server
12
13
  end
13
14
 
15
+ TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/plugin_helper_server")
16
+
14
17
  PORT = unused_port
15
18
 
16
19
  setup do
@@ -27,12 +30,12 @@ class ServerPluginHelperTest < Test::Unit::TestCase
27
30
  end
28
31
 
29
32
  teardown do
30
- @d.stopped? || @d.stop
31
- @d.before_shutdown? || @d.before_shutdown
32
- @d.shutdown? || @d.shutdown
33
- @d.after_shutdown? || @d.after_shutdown
34
- @d.closed? || @d.close
35
- @d.terminated? || @d.terminate
33
+ (@d.stopped? || @d.stop) rescue nil
34
+ (@d.before_shutdown? || @d.before_shutdown) rescue nil
35
+ (@d.shutdown? || @d.shutdown) rescue nil
36
+ (@d.after_shutdown? || @d.after_shutdown) rescue nil
37
+ (@d.closed? || @d.close) rescue nil
38
+ (@d.terminated? || @d.terminate) rescue nil
36
39
 
37
40
  @socket_manager_server.close
38
41
  if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
@@ -141,11 +144,49 @@ class ServerPluginHelperTest < Test::Unit::TestCase
141
144
  assert created_server.is_a?(Coolio::TCPServer)
142
145
  end
143
146
 
144
- # tests about "proto: :udp" is in #server_create
147
+ data(methods)
148
+ test 'creates tls server in default if transport section and tcp protocol specified' do |m|
149
+ @d = d = Dummy.new
150
+ transport_conf = config_element('transport', 'tcp', {}, [])
151
+ d.configure(config_element('ROOT', '', {}, [transport_conf]))
152
+ d.start
153
+ d.after_start
154
+
155
+ d.__send__(m, :myserver, PORT){|x| x }
156
+
157
+ created_server_info = @d._servers.first
158
+ assert_equal :tcp, created_server_info.proto
159
+ created_server = created_server_info.server
160
+ assert created_server.is_a?(Coolio::TCPServer)
161
+ end
145
162
 
146
163
  data(methods)
147
164
  test 'creates tls server if specified in proto' do |m|
148
- # pend "not implemented yet"
165
+ assert_raise(ArgumentError.new("BUG: TLS transport specified, but certification options are not specified")) do
166
+ @d.__send__(m, :myserver, PORT, proto: :tls){|x| x }
167
+ end
168
+ @d.__send__(m, :myserver, PORT, proto: :tls, tls_options: {insecure: true}){|x| x }
169
+
170
+ created_server_info = @d._servers.first
171
+ assert_equal :tls, created_server_info.proto
172
+ created_server = created_server_info.server
173
+ assert created_server.is_a?(Coolio::TCPServer) # yes, TCP here
174
+ end
175
+
176
+ data(methods)
177
+ test 'creates tls server in default if transport section and tls protocol specified' do |m|
178
+ @d = d = Dummy.new
179
+ transport_conf = config_element('transport', 'tls', {'insecure' => 'true'}, [])
180
+ d.configure(config_element('ROOT', '', {}, [transport_conf]))
181
+ d.start
182
+ d.after_start
183
+
184
+ d.__send__(m, :myserver, PORT){|x| x }
185
+
186
+ created_server_info = @d._servers.first
187
+ assert_equal :tls, created_server_info.proto
188
+ created_server = created_server_info.server
189
+ assert created_server.is_a?(Coolio::TCPServer) # OK, it's Coolio::TCPServer
149
190
  end
150
191
 
151
192
  data(methods)
@@ -162,10 +203,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
162
203
 
163
204
  data(
164
205
  'server_create tcp' => [:server_create, :tcp],
165
- # 'server_create tls' => [:server_create, :tls],
206
+ 'server_create tls' => [:server_create, :tls],
166
207
  # 'server_create unix' => [:server_create, :unix],
167
208
  'server_create_connection tcp' => [:server_create_connection, :tcp],
168
- # 'server_create_connection tcp' => [:server_create_connection, :tls],
209
+ 'server_create_connection tls' => [:server_create_connection, :tls],
169
210
  # 'server_create_connection tcp' => [:server_create_connection, :unix],
170
211
  )
171
212
  test 'raise error if udp options specified for tcp/tls/unix' do |(m, proto)|
@@ -203,17 +244,17 @@ class ServerPluginHelperTest < Test::Unit::TestCase
203
244
  # 'server_create_connection unix' => [:server_create_connection, :unix, {}],
204
245
  )
205
246
  test 'raise error if tls options specified for tcp/udp/unix' do |(m, proto, kwargs)|
206
- assert_raise(ArgumentError.new("BUG: certopts is available only for tls")) do
207
- @d.__send__(m, :myserver, PORT, proto: proto, certopts: {}, **kwargs){|x| x }
247
+ assert_raise(ArgumentError.new("BUG: tls_options is available only for tls")) do
248
+ @d.__send__(m, :myserver, PORT, proto: proto, tls_options: {}, **kwargs){|x| x }
208
249
  end
209
250
  end
210
251
 
211
252
  data(
212
253
  'server_create tcp' => [:server_create, :tcp, {}],
213
254
  'server_create udp' => [:server_create, :udp, {max_bytes: 128}],
214
- # 'server_create tls' => [:server_create, :tls, {}],
255
+ 'server_create tls' => [:server_create, :tls, {tls_options: {insecure: true}}],
215
256
  'server_create_connection tcp' => [:server_create_connection, :tcp, {}],
216
- # 'server_create_connection tls' => [:server_create_connection, :tls, {}],
257
+ 'server_create_connection tls' => [:server_create_connection, :tls, {tls_options: {insecure: true}}],
217
258
  )
218
259
  test 'can bind specified IPv4 address' do |(m, proto, kwargs)|
219
260
  @d.__send__(m, :myserver, PORT, proto: proto, bind: "127.0.0.1", **kwargs){|x| x }
@@ -224,9 +265,9 @@ class ServerPluginHelperTest < Test::Unit::TestCase
224
265
  data(
225
266
  'server_create tcp' => [:server_create, :tcp, {}],
226
267
  'server_create udp' => [:server_create, :udp, {max_bytes: 128}],
227
- # 'server_create tls' => [:server_create, :tls, {}],
268
+ 'server_create tls' => [:server_create, :tls, {tls_options: {insecure: true}}],
228
269
  'server_create_connection tcp' => [:server_create_connection, :tcp, {}],
229
- # 'server_create_connection tls' => [:server_create_connection, :tls, {}],
270
+ 'server_create_connection tls' => [:server_create_connection, :tls, {tls_options: {insecure: true}}],
230
271
  )
231
272
  test 'can bind specified IPv6 address' do |(m, proto, kwargs)| # if available
232
273
  omit "IPv6 unavailable here" unless ipv6_enabled?
@@ -238,10 +279,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
238
279
  data(
239
280
  'server_create tcp' => [:server_create, :tcp, {}],
240
281
  'server_create udp' => [:server_create, :udp, {max_bytes: 128}],
241
- # 'server_create tls' => [:server_create, :tls, {}],
282
+ 'server_create tls' => [:server_create, :tls, {tls_options: {insecure: true}}],
242
283
  # 'server_create unix' => [:server_create, :unix, {}],
243
284
  'server_create_connection tcp' => [:server_create, :tcp, {}],
244
- # 'server_create_connection tls' => [:server_create, :tls, {}],
285
+ 'server_create_connection tls' => [:server_create, :tls, {tls_options: {insecure: true}}],
245
286
  # 'server_create_connection unix' => [:server_create, :unix, {}],
246
287
  )
247
288
  test 'can create 2 or more servers which share same bind address and port if shared option is true' do |(m, proto, kwargs)|
@@ -260,10 +301,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
260
301
  data(
261
302
  'server_create tcp' => [:server_create, :tcp, {}],
262
303
  'server_create udp' => [:server_create, :udp, {max_bytes: 128}],
263
- # 'server_create tls' => [:server_create, :tls, {}],
304
+ 'server_create tls' => [:server_create, :tls, {tls_options: {insecure: true}}],
264
305
  # 'server_create unix' => [:server_create, :unix, {}],
265
306
  'server_create_connection tcp' => [:server_create, :tcp, {}],
266
- # 'server_create_connection tls' => [:server_create, :tls, {}],
307
+ 'server_create_connection tls' => [:server_create, :tls, {tls_options: {insecure: true}}],
267
308
  # 'server_create_connection unix' => [:server_create, :unix, {}],
268
309
  )
269
310
  test 'cannot create 2 or more servers using same bind address and port if shared option is false' do |(m, proto, kwargs)|
@@ -286,7 +327,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
286
327
  data(
287
328
  'tcp' => [:tcp, {}],
288
329
  'udp' => [:udp, {max_bytes: 128}],
289
- # 'tls' => [:tls, {}],
330
+ 'tls' => [:tls, {tls_options: {insecure: true}}],
290
331
  # 'unix' => [:unix, {}],
291
332
  )
292
333
  test 'raise error if block argument is not specified or too many' do |(proto, kwargs)|
@@ -428,7 +469,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
428
469
  end
429
470
  waiting(10){ sleep 0.1 until received.bytesize == 4 || errors.size == 1 }
430
471
  assert_equal "foo\n", received
431
- assert_equal 1, errors.size
472
+ assert{ errors.size > 0 } # it might be called twice (or more) when connection was accepted, and then data arrived (or more)
432
473
  assert_equal "data callback can be registered just once, but registered twice", errors.first.message
433
474
  end
434
475
 
@@ -645,7 +686,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
645
686
  sock.write "foo\n"
646
687
  sock.close
647
688
 
648
- waiting(10){ sleep 0.1 until received.bytesize == 4 || errors.size == 1 }
689
+ waiting(10){ sleep 0.1 until received.bytesize == 4 && errors.size == 1 }
649
690
  assert_equal "foo\n", received
650
691
  assert_equal 1, errors.size
651
692
  assert_equal "BUG: this event is disabled for udp: data", errors.first.message
@@ -667,7 +708,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
667
708
  sock.write "foo\n"
668
709
  sock.close
669
710
 
670
- waiting(10){ sleep 0.1 until received.bytesize == 4 || errors.size == 1 }
711
+ waiting(10){ sleep 0.1 until received.bytesize == 4 && errors.size == 1 }
671
712
  assert_equal "foo\n", received
672
713
  assert_equal 1, errors.size
673
714
  assert_equal "BUG: this event is disabled for udp: write_complete", errors.first.message
@@ -689,32 +730,665 @@ class ServerPluginHelperTest < Test::Unit::TestCase
689
730
  sock.write "foo\n"
690
731
  sock.close
691
732
 
692
- waiting(10){ sleep 0.1 until received.bytesize == 4 || errors.size == 1 }
733
+ waiting(10){ sleep 0.1 until received.bytesize == 4 && errors.size == 1 }
693
734
  assert_equal "foo\n", received
694
735
  assert_equal 1, errors.size
695
736
  assert_equal "BUG: this event is disabled for udp: close", errors.first.message
696
737
  end
697
738
  end
698
739
 
740
+ module CertUtil
741
+ extend Fluent::PluginHelper::CertOption
742
+ end
743
+
744
+ def create_ca_options
745
+ {
746
+ private_key_length: 2048,
747
+ country: 'US',
748
+ state: 'CA',
749
+ locality: 'Mountain View',
750
+ common_name: 'ca.testing.fluentd.org',
751
+ expiration: 30 * 86400,
752
+ digest: :sha256,
753
+ }
754
+ end
755
+
756
+ def create_server_options
757
+ {
758
+ private_key_length: 2048,
759
+ country: 'US',
760
+ state: 'CA',
761
+ locality: 'Mountain View',
762
+ common_name: 'server.testing.fluentd.org',
763
+ expiration: 30 * 86400,
764
+ digest: :sha256,
765
+ }
766
+ end
767
+
768
+ def write_cert_and_key(cert_path, cert, key_path, key, passphrase)
769
+ File.open(cert_path, "w"){|f| f.write(cert.to_pem) }
770
+ # Encrypt secret key by AES256, and write it in PEM format
771
+ File.open(key_path, "w"){|f| f.write(key.export(OpenSSL::Cipher.new("AES-256-CBC"), passphrase)) }
772
+ File.chmod(0600, cert_path, key_path)
773
+ end
774
+
775
+ def create_server_pair_signed_by_self(cert_path, private_key_path, passphrase)
776
+ cert, key, _ = CertUtil.cert_option_generate_server_pair_self_signed(create_server_options)
777
+ write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
778
+ end
779
+
780
+ def create_ca_pair_signed_by_self(cert_path, private_key_path, passphrase)
781
+ cert, key, _ = CertUtil.cert_option_generate_ca_pair_self_signed(create_ca_options)
782
+ write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
783
+ end
784
+
785
+ def create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)
786
+ cert, key, _ = CertUtil.cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, create_server_options)
787
+ write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
788
+ end
789
+
790
+ def create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)
791
+ root_cert, root_key, _ = CertUtil.cert_option_generate_ca_pair_self_signed(create_ca_options)
792
+ write_cert_and_key(ca_cert_path, root_cert, ca_key_path, root_key, ca_key_passphrase)
793
+
794
+ intermediate_ca_options = create_ca_options
795
+ intermediate_ca_options[:common_name] = 'ca2.testing.fluentd.org'
796
+ chain_cert, chain_key = CertUtil.cert_option_generate_pair(intermediate_ca_options, root_cert.subject)
797
+ chain_cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
798
+ chain_cert.sign(root_key, "sha256")
799
+
800
+ server_cert, server_key, _ = CertUtil.cert_option_generate_pair(create_server_options, chain_cert.subject)
801
+ server_cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
802
+ server_cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
803
+ server_cert.sign(chain_key, "sha256")
804
+
805
+ # write chained cert
806
+ File.open(cert_path, "w") do |f|
807
+ f.write server_cert.to_pem
808
+ f.write chain_cert.to_pem
809
+ end
810
+ File.open(private_key_path, "w"){|f| f.write(server_key.export(OpenSSL::Cipher.new("AES-256-CBC"), passphrase)) }
811
+ File.chmod(0600, cert_path, private_key_path)
812
+ end
813
+
814
+ def open_tls_session(addr, port, verify: true, cert_path: nil, selfsigned: true, hostname: nil)
815
+ context = OpenSSL::SSL::SSLContext.new
816
+ context.set_params({})
817
+ if verify
818
+ cert_store = OpenSSL::X509::Store.new
819
+ cert_store.set_default_paths
820
+ if selfsigned && OpenSSL::X509.const_defined?('V_FLAG_CHECK_SS_SIGNATURE')
821
+ cert_store.flags = OpenSSL::X509::V_FLAG_CHECK_SS_SIGNATURE
822
+ end
823
+ if cert_path
824
+ cert_store.add_file(cert_path)
825
+ end
826
+ context.verify_mode = OpenSSL::SSL::VERIFY_PEER
827
+ context.cert_store = cert_store
828
+ if !hostname && context.respond_to?(:verify_hostname=)
829
+ context.verify_hostname = false # In test code, using hostname to be connected is very difficult
830
+ end
831
+ else
832
+ context.verify_mode = OpenSSL::SSL::VERIFY_NONE
833
+ end
834
+
835
+ sock = OpenSSL::SSL::SSLSocket.new(TCPSocket.new(addr, port), context)
836
+ sock.hostname = hostname if hostname && sock.respond_to?(:hostname)
837
+ sock.connect
838
+ yield sock
839
+ ensure
840
+ sock.close rescue nil
841
+ end
842
+
843
+ sub_test_case '#server_create_tls with various certificate options' do
844
+ setup do
845
+ @d = Dummy.new # to get plugin not configured/started yet
846
+
847
+ @certs_dir = File.join(TMP_DIR, "tls_certs")
848
+ @server_cert_dir = File.join(@certs_dir, "server")
849
+ FileUtils.rm_rf @certs_dir
850
+ FileUtils.mkdir_p @server_cert_dir
851
+ end
852
+
853
+ sub_test_case 'using tls_options arguments to specify cert options' do
854
+ setup do
855
+ @d.configure(config_element()); @d.start; @d.after_start
856
+ end
857
+
858
+ test 'create dynamic self-signed cert/key pair (without any verification from clients)' do
859
+ # insecure
860
+ tls_options = {
861
+ protocol: :tls,
862
+ version: 'TLSv1_2',
863
+ ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',
864
+ insecure: true,
865
+ generate_private_key_length: 2048,
866
+ generate_cert_country: 'US',
867
+ generate_cert_state: 'CA',
868
+ generate_cert_locality: 'Mountain View',
869
+ generate_cert_common_name: 'myserver.testing.fluentd.org',
870
+ generate_cert_expiration: 10 * 365 * 86400,
871
+ generate_cert_digest: :sha256,
872
+ }
873
+
874
+ received = ""
875
+ @d.server_create_tls(:s, PORT, tls_options: tls_options) do |data, conn|
876
+ received << data
877
+ end
878
+ assert_raise "" do
879
+ open_tls_session('127.0.0.1', PORT) do |sock|
880
+ sock.post_connection_check('myserver.testing.fluentd.org')
881
+ # cannot connect ....
882
+ end
883
+ end
884
+ open_tls_session('127.0.0.1', PORT, verify: false) do |sock|
885
+ sock.puts "yay"
886
+ sock.puts "foo"
887
+ end
888
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
889
+ assert_equal "yay\nfoo\n", received
890
+ end
891
+
892
+ test 'load self-signed cert/key pair (files), verified from clients using cert files' do
893
+ cert_path = File.join(@server_cert_dir, "cert.pem")
894
+ private_key_path = File.join(@certs_dir, "server.key.pem")
895
+ private_key_passphrase = "yaaaaaaaaaaaaaaaaaaay"
896
+ create_server_pair_signed_by_self(cert_path, private_key_path, private_key_passphrase)
897
+
898
+ tls_options = {
899
+ protocol: :tls,
900
+ version: 'TLSv1_2',
901
+ ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',
902
+ insecure: false,
903
+ cert_path: cert_path,
904
+ private_key_path: private_key_path,
905
+ private_key_passphrase: private_key_passphrase,
906
+ }
907
+ received = ""
908
+ @d.server_create_tls(:s, PORT, tls_options: tls_options) do |data, conn|
909
+ received << data
910
+ end
911
+ assert_raise "" do
912
+ open_tls_session('127.0.0.1', PORT) do |sock|
913
+ sock.post_connection_check('server.testing.fluentd.org')
914
+ # cannot connect by failing verification without server cert
915
+ end
916
+ end
917
+ open_tls_session('127.0.0.1', PORT, cert_path: cert_path) do |sock|
918
+ sock.puts "yay"
919
+ sock.puts "foo"
920
+ end
921
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
922
+ assert_equal "yay\nfoo\n", received
923
+ end
924
+
925
+ test 'create dynamic server cert by private CA cert file, verified from clients using CA cert file' do
926
+ ca_cert_path = File.join(@certs_dir, "ca_cert.pem")
927
+ ca_key_path = File.join(@certs_dir, "ca.key.pem")
928
+ ca_key_passphrase = "fooooooooooooooooooooooooo"
929
+ create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, ca_key_passphrase)
930
+
931
+ tls_options = {
932
+ protocol: :tls,
933
+ version: 'TLSv1_2',
934
+ ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',
935
+ insecure: false,
936
+ ca_cert_path: ca_cert_path,
937
+ ca_private_key_path: ca_key_path,
938
+ ca_private_key_passphrase: ca_key_passphrase,
939
+ generate_private_key_length: 2048,
940
+ }
941
+ received = ""
942
+ @d.server_create_tls(:s, PORT, tls_options: tls_options) do |data, conn|
943
+ received << data
944
+ end
945
+ open_tls_session('127.0.0.1', PORT, cert_path: ca_cert_path) do |sock|
946
+ sock.puts "yay"
947
+ sock.puts "foo"
948
+ end
949
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
950
+ assert_equal "yay\nfoo\n", received
951
+ end
952
+
953
+ test 'load static server cert by private CA cert file, verified from clients using CA cert file' do
954
+ ca_cert_path = File.join(@certs_dir, "ca_cert.pem")
955
+ ca_key_path = File.join(@certs_dir, "ca.key.pem")
956
+ ca_key_passphrase = "foooooooo"
957
+ create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, ca_key_passphrase)
958
+
959
+ cert_path = File.join(@server_cert_dir, "cert.pem")
960
+ private_key_path = File.join(@certs_dir, "server.key.pem")
961
+ private_key_passphrase = "yaaaaaaaaaaaaaaaaaaay"
962
+ create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)
963
+
964
+ tls_options = {
965
+ protocol: :tls,
966
+ version: 'TLSv1_2',
967
+ ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',
968
+ insecure: false,
969
+ cert_path: cert_path,
970
+ private_key_path: private_key_path,
971
+ private_key_passphrase: private_key_passphrase,
972
+ }
973
+ received = ""
974
+ @d.server_create_tls(:s, PORT, tls_options: tls_options) do |data, conn|
975
+ received << data
976
+ end
977
+ open_tls_session('127.0.0.1', PORT, cert_path: ca_cert_path) do |sock|
978
+ sock.puts "yay"
979
+ sock.puts "foo"
980
+ end
981
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
982
+ assert_equal "yay\nfoo\n", received
983
+ end
984
+
985
+ test 'load chained server cert by private CA cert file, verified from clients using CA cert file as root' do
986
+ ca_cert_path = File.join(@certs_dir, "ca_cert.pem")
987
+ ca_key_path = File.join(@certs_dir, "ca.key.pem")
988
+ ca_key_passphrase = "foooooooo"
989
+ cert_path = File.join(@server_cert_dir, "cert.pem")
990
+ private_key_path = File.join(@certs_dir, "server.key.pem")
991
+ private_key_passphrase = "yaaaaaaaaaaaaaaaaaaay"
992
+ create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)
993
+
994
+ tls_options = {
995
+ protocol: :tls,
996
+ version: 'TLSv1_2',
997
+ ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',
998
+ insecure: false,
999
+ cert_path: cert_path,
1000
+ private_key_path: private_key_path,
1001
+ private_key_passphrase: private_key_passphrase,
1002
+ }
1003
+ received = ""
1004
+ @d.server_create_tls(:s, PORT, tls_options: tls_options) do |data, conn|
1005
+ received << data
1006
+ end
1007
+ open_tls_session('127.0.0.1', PORT, cert_path: ca_cert_path) do |sock|
1008
+ sock.puts "yay"
1009
+ sock.puts "foo"
1010
+ end
1011
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
1012
+ assert_equal "yay\nfoo\n", received
1013
+ end
1014
+ end
1015
+
1016
+ sub_test_case 'using configurations to specify cert options' do
1017
+ test 'create dynamic self-signed cert/key pair (without any verification from clients)' do
1018
+ # insecure
1019
+ transport_opts = {
1020
+ 'insecure' => 'true',
1021
+ }
1022
+ transport_conf = config_element('transport', 'tls', transport_opts)
1023
+ conf = config_element('match', 'tag.*', {}, [transport_conf])
1024
+
1025
+ @d.configure(conf); @d.start; @d.after_start
1026
+
1027
+ received = ""
1028
+ @d.server_create_tls(:s, PORT) do |data, conn|
1029
+ received << data
1030
+ end
1031
+ assert_raise "" do
1032
+ open_tls_session('127.0.0.1', PORT) do |sock|
1033
+ sock.post_connection_check('myserver.testing.fluentd.org')
1034
+ # cannot connect ....
1035
+ end
1036
+ end
1037
+ open_tls_session('127.0.0.1', PORT, verify: false) do |sock|
1038
+ sock.puts "yay"
1039
+ sock.puts "foo"
1040
+ end
1041
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
1042
+ assert_equal "yay\nfoo\n", received
1043
+ end
1044
+
1045
+ test 'load self-signed cert/key pair (files), verified from clients using cert files' do
1046
+ cert_path = File.join(@server_cert_dir, "cert.pem")
1047
+ private_key_path = File.join(@certs_dir, "server.key.pem")
1048
+ private_key_passphrase = "yaaaaaaaaaaaaaaaaaaay"
1049
+ create_server_pair_signed_by_self(cert_path, private_key_path, private_key_passphrase)
1050
+
1051
+ transport_opts = {
1052
+ 'cert_path' => cert_path,
1053
+ 'private_key_path' => private_key_path,
1054
+ 'private_key_passphrase' => private_key_passphrase,
1055
+ }
1056
+ transport_conf = config_element('transport', 'tls', transport_opts)
1057
+ conf = config_element('match', 'tag.*', {}, [transport_conf])
1058
+
1059
+ @d.configure(conf); @d.start; @d.after_start
1060
+
1061
+ received = ""
1062
+ @d.server_create_tls(:s, PORT) do |data, conn|
1063
+ received << data
1064
+ end
1065
+ assert_raise "" do
1066
+ open_tls_session('127.0.0.1', PORT) do |sock|
1067
+ sock.post_connection_check('server.testing.fluentd.org')
1068
+ # cannot connect by failing verification without server cert
1069
+ end
1070
+ end
1071
+ open_tls_session('127.0.0.1', PORT, cert_path: cert_path) do |sock|
1072
+ sock.puts "yay"
1073
+ sock.puts "foo"
1074
+ end
1075
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
1076
+ assert_equal "yay\nfoo\n", received
1077
+ end
1078
+
1079
+ test 'create dynamic server cert by private CA cert file, verified from clients using CA cert file' do
1080
+ ca_cert_path = File.join(@certs_dir, "ca_cert.pem")
1081
+ ca_key_path = File.join(@certs_dir, "ca.key.pem")
1082
+ ca_key_passphrase = "fooooooooooooooooooooooooo"
1083
+ create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, ca_key_passphrase)
1084
+
1085
+ transport_opts = {
1086
+ 'ca_cert_path' => ca_cert_path,
1087
+ 'ca_private_key_path' => ca_key_path,
1088
+ 'ca_private_key_passphrase' => ca_key_passphrase,
1089
+ }
1090
+ transport_conf = config_element('transport', 'tls', transport_opts)
1091
+ conf = config_element('match', 'tag.*', {}, [transport_conf])
1092
+
1093
+ @d.configure(conf); @d.start; @d.after_start
1094
+
1095
+ received = ""
1096
+ @d.server_create_tls(:s, PORT) do |data, conn|
1097
+ received << data
1098
+ end
1099
+ open_tls_session('127.0.0.1', PORT, cert_path: ca_cert_path) do |sock|
1100
+ sock.puts "yay"
1101
+ sock.puts "foo"
1102
+ end
1103
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
1104
+ assert_equal "yay\nfoo\n", received
1105
+ end
1106
+
1107
+ test 'load static server cert by private CA cert file, verified from clients using CA cert file' do
1108
+ ca_cert_path = File.join(@certs_dir, "ca_cert.pem")
1109
+ ca_key_path = File.join(@certs_dir, "ca.key.pem")
1110
+ ca_key_passphrase = "foooooooo"
1111
+ create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, ca_key_passphrase)
1112
+
1113
+ cert_path = File.join(@server_cert_dir, "cert.pem")
1114
+ private_key_path = File.join(@certs_dir, "server.key.pem")
1115
+ private_key_passphrase = "yaaaaaaaaaaaaaaaaaaay"
1116
+ create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)
1117
+
1118
+ transport_opts = {
1119
+ 'cert_path' => cert_path,
1120
+ 'private_key_path' => private_key_path,
1121
+ 'private_key_passphrase' => private_key_passphrase,
1122
+ }
1123
+ transport_conf = config_element('transport', 'tls', transport_opts)
1124
+ conf = config_element('match', 'tag.*', {}, [transport_conf])
1125
+
1126
+ @d.configure(conf); @d.start; @d.after_start
1127
+
1128
+ received = ""
1129
+ @d.server_create_tls(:s, PORT) do |data, conn|
1130
+ received << data
1131
+ end
1132
+ open_tls_session('127.0.0.1', PORT, cert_path: ca_cert_path) do |sock|
1133
+ sock.puts "yay"
1134
+ sock.puts "foo"
1135
+ end
1136
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
1137
+ assert_equal "yay\nfoo\n", received
1138
+ end
1139
+
1140
+ test 'load chained server cert by private CA cert file, verified from clients using CA cert file as root' do
1141
+ ca_cert_path = File.join(@certs_dir, "ca_cert.pem")
1142
+ ca_key_path = File.join(@certs_dir, "ca.key.pem")
1143
+ ca_key_passphrase = "foooooooo"
1144
+ cert_path = File.join(@server_cert_dir, "cert.pem")
1145
+ private_key_path = File.join(@certs_dir, "server.key.pem")
1146
+ private_key_passphrase = "yaaaaaaaaaaaaaaaaaaay"
1147
+ create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)
1148
+
1149
+ transport_opts = {
1150
+ 'cert_path' => cert_path,
1151
+ 'private_key_path' => private_key_path,
1152
+ 'private_key_passphrase' => private_key_passphrase,
1153
+ }
1154
+ transport_conf = config_element('transport', 'tls', transport_opts)
1155
+ conf = config_element('match', 'tag.*', {}, [transport_conf])
1156
+
1157
+ @d.configure(conf); @d.start; @d.after_start
1158
+
1159
+ received = ""
1160
+ @d.server_create_tls(:s, PORT) do |data, conn|
1161
+ received << data
1162
+ end
1163
+ open_tls_session('127.0.0.1', PORT, cert_path: ca_cert_path) do |sock|
1164
+ sock.puts "yay"
1165
+ sock.puts "foo"
1166
+ end
1167
+ waiting(10){ sleep 0.1 until received.bytesize == 8 }
1168
+ assert_equal "yay\nfoo\n", received
1169
+ end
1170
+ end
1171
+ end
1172
+
699
1173
  sub_test_case '#server_create_tls' do
700
- # not implemented yet
1174
+ setup do
1175
+ @certs_dir = File.join(TMP_DIR, "tls_certs")
1176
+ FileUtils.rm_rf @certs_dir
1177
+ FileUtils.mkdir_p @certs_dir
1178
+
1179
+ @server_cert_dir = File.join(@certs_dir, "server")
1180
+ FileUtils.mkdir_p @server_cert_dir
1181
+
1182
+ @cert_path = File.join(@server_cert_dir, "cert.pem")
1183
+ private_key_path = File.join(@certs_dir, "server.key.pem")
1184
+ private_key_passphrase = "yaaaaaaaaaaaaaaaaaaay"
1185
+ create_server_pair_signed_by_self(@cert_path, private_key_path, private_key_passphrase)
1186
+
1187
+ @default_hostname = ::Socket.gethostname
1188
+
1189
+ @tls_options = {
1190
+ protocol: :tls,
1191
+ version: 'TLSv1_2',
1192
+ ciphers: 'ALL:!aNULL:!eNULL:!SSLv2',
1193
+ insecure: false,
1194
+ cert_path: @cert_path,
1195
+ private_key_path: private_key_path,
1196
+ private_key_passphrase: private_key_passphrase,
1197
+ }
1198
+ end
701
1199
 
702
- # test 'can accept all keyword arguments valid for tcp/tls server'
703
- # test 'creates a tls server just to read data'
704
- # test 'creates a tls server to read and write data'
705
- # test 'creates a tls server to read and write data using IPv6'
1200
+ test 'can accept all keyword arguments valid for tcp/tls server' do
1201
+ assert_nothing_raised do
1202
+ @d.server_create_tls(:s, PORT, bind: '127.0.0.1', shared: false, resolve_name: true, linger_timeout: 10, backlog: 500, tls_options: @tls_options) do |data, conn|
1203
+ # ...
1204
+ end
1205
+ end
1206
+ end
1207
+
1208
+ test 'creates a tls server just to read data' do
1209
+ received = ""
1210
+ @d.server_create_tls(:s, PORT, tls_options: @tls_options) do |data, conn|
1211
+ received << data
1212
+ end
1213
+ 3.times do
1214
+ open_tls_session('127.0.0.1', PORT, cert_path: @cert_path) do |sock|
1215
+ sock.puts "yay"
1216
+ sock.puts "foo"
1217
+ end
1218
+ end
1219
+ waiting(10){ sleep 0.1 until received.bytesize == 24 }
1220
+ assert_equal 3, received.scan("yay\n").size
1221
+ assert_equal 3, received.scan("foo\n").size
1222
+ end
706
1223
 
707
- # many tests about certops
1224
+ test 'creates a tls server to read and write data' do
1225
+ received = ""
1226
+ responses = []
1227
+ @d.server_create_tls(:s, PORT, tls_options: @tls_options) do |data, conn|
1228
+ received << data
1229
+ conn.write "ack\n"
1230
+ end
1231
+ 3.times do
1232
+ # open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, hostname: @default_hostname) do |sock|
1233
+ open_tls_session('127.0.0.1', PORT, cert_path: @cert_path) do |sock|
1234
+ sock.puts "yay"
1235
+ sock.puts "foo"
1236
+ responses << sock.readline
1237
+ end
1238
+ end
1239
+ waiting(10){ sleep 0.1 until received.bytesize == 24 }
1240
+ assert_equal 3, received.scan("yay\n").size
1241
+ assert_equal 3, received.scan("foo\n").size
1242
+ assert_equal ["ack\n","ack\n","ack\n"], responses
1243
+ end
708
1244
 
709
- # test 'does not resolve name of client address in default'
710
- # test 'does resolve name of client address if resolve_name is true'
711
- # test 'can keep connections alive for tls if keepalive specified' do
712
- # pend "not implemented yet"
713
- # end
1245
+ test 'creates a tls server to read and write data using IPv6' do
1246
+ omit "IPv6 unavailable here" unless ipv6_enabled?
714
1247
 
715
- # test 'raises error if plugin registers data callback for connection object from #server_create'
716
- # test 'can call write_complete callback if registered'
717
- # test 'can call close callback if registered'
1248
+ received = ""
1249
+ responses = []
1250
+ @d.server_create_tls(:s, PORT, bind: "::1", tls_options: @tls_options) do |data, conn|
1251
+ received << data
1252
+ conn.write "ack\n"
1253
+ end
1254
+ 3.times do
1255
+ # open_tls_session('::1', PORT, cert_path: @cert_path, hostname: @default_hostname) do |sock|
1256
+ open_tls_session('::1', PORT, cert_path: @cert_path) do |sock|
1257
+ sock.puts "yay"
1258
+ sock.puts "foo"
1259
+ responses << sock.readline
1260
+ end
1261
+ end
1262
+ waiting(10){ sleep 0.1 until received.bytesize == 24 }
1263
+ assert_equal 3, received.scan("yay\n").size
1264
+ assert_equal 3, received.scan("foo\n").size
1265
+ assert_equal ["ack\n","ack\n","ack\n"], responses
1266
+ end
1267
+
1268
+ test 'does not resolve name of client address in default' do
1269
+ received = ""
1270
+ sources = []
1271
+ @d.server_create_tls(:s, PORT, tls_options: @tls_options) do |data, conn|
1272
+ received << data
1273
+ sources << conn.remote_host
1274
+ end
1275
+ 3.times do
1276
+ # open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, hostname: @default_hostname) do |sock|
1277
+ open_tls_session('127.0.0.1', PORT, cert_path: @cert_path) do |sock|
1278
+ sock.puts "yay"
1279
+ end
1280
+ end
1281
+ waiting(10){ sleep 0.1 until received.bytesize == 12 }
1282
+ assert_equal 3, received.scan("yay\n").size
1283
+ assert{ sources.all?{|s| s == "127.0.0.1" } }
1284
+ end
1285
+
1286
+ test 'does resolve name of client address if resolve_name is true' do
1287
+ hostname = Socket.getnameinfo([nil, nil, nil, "127.0.0.1"])[0]
1288
+
1289
+ received = ""
1290
+ sources = []
1291
+ @d.server_create_tls(:s, PORT, resolve_name: true, tls_options: @tls_options) do |data, conn|
1292
+ received << data
1293
+ sources << conn.remote_host
1294
+ end
1295
+ 3.times do
1296
+ # open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, hostname: @default_hostname) do |sock|
1297
+ open_tls_session('127.0.0.1', PORT, cert_path: @cert_path) do |sock|
1298
+ sock.puts "yay"
1299
+ end
1300
+ end
1301
+ waiting(10){ sleep 0.1 until received.bytesize == 12 }
1302
+ assert_equal 3, received.scan("yay\n").size
1303
+ assert{ sources.all?{|s| s == hostname } }
1304
+ end
1305
+
1306
+ test 'can keep connections alive for tls if keepalive specified' do
1307
+ # pend "not implemented yet"
1308
+ end
1309
+
1310
+ test 'raises error if plugin registers data callback for connection object from #server_create' do
1311
+ received = ""
1312
+ errors = []
1313
+ @d.server_create_tls(:s, PORT, tls_options: @tls_options) do |data, conn|
1314
+ received << data
1315
+ begin
1316
+ conn.data{|d| received << d.upcase }
1317
+ rescue => e
1318
+ errors << e
1319
+ end
1320
+ end
1321
+ open_tls_session('127.0.0.1', PORT, cert_path: @cert_path) do |sock|
1322
+ sock.puts "foo"
1323
+ end
1324
+ waiting(10){ sleep 0.1 until received.bytesize == 4 || errors.size == 1 }
1325
+ assert_equal "foo\n", received
1326
+ assert_equal 1, errors.size
1327
+ assert_equal "data callback can be registered just once, but registered twice", errors.first.message
1328
+ end
1329
+
1330
+ test 'can call write_complete callback if registered' do
1331
+ buffer = ""
1332
+ lines = []
1333
+ responses = []
1334
+ response_completes = []
1335
+ @d.server_create_tls(:s, PORT, tls_options: @tls_options) do |data, conn|
1336
+ conn.on(:write_complete){|c| response_completes << true }
1337
+ buffer << data
1338
+ if idx = buffer.index("\n")
1339
+ lines << buffer.slice!(0,idx+1)
1340
+ conn.write "ack\n"
1341
+ end
1342
+ end
1343
+ 3.times do
1344
+ open_tls_session('127.0.0.1', PORT, cert_path: @cert_path) do |sock|
1345
+ sock.write "yay"
1346
+ sock.write "foo\n"
1347
+ begin
1348
+ responses << sock.readline
1349
+ rescue EOFError, IOError, Errno::ECONNRESET
1350
+ # ignore
1351
+ end
1352
+ sock.close
1353
+ end
1354
+ end
1355
+ waiting(10){ sleep 0.1 until lines.size == 3 && response_completes.size == 3 }
1356
+ assert_equal ["yayfoo\n", "yayfoo\n", "yayfoo\n"], lines
1357
+ assert_equal ["ack\n","ack\n","ack\n"], responses
1358
+ assert_equal [true, true, true], response_completes
1359
+ end
1360
+
1361
+ test 'can call close callback if registered' do
1362
+ buffer = ""
1363
+ lines = []
1364
+ callback_results = []
1365
+ @d.server_create_tls(:s, PORT, tls_options: @tls_options) do |data, conn|
1366
+ conn.on(:close){|c| callback_results << "closed" }
1367
+ buffer << data
1368
+ if idx = buffer.index("\n")
1369
+ lines << buffer.slice!(0,idx+1)
1370
+ conn.write "ack\n"
1371
+ end
1372
+ end
1373
+ 3.times do
1374
+ open_tls_session('127.0.0.1', PORT, cert_path: @cert_path) do |sock|
1375
+ sock.write "yay"
1376
+ sock.write "foo\n"
1377
+ begin
1378
+ while line = sock.readline
1379
+ if line == "ack\n"
1380
+ sock.close
1381
+ end
1382
+ end
1383
+ rescue EOFError, IOError, Errno::ECONNRESET
1384
+ # ignore
1385
+ end
1386
+ end
1387
+ end
1388
+ waiting(10){ sleep 0.1 until lines.size == 3 && callback_results.size == 3 }
1389
+ assert_equal ["yayfoo\n", "yayfoo\n", "yayfoo\n"], lines
1390
+ assert_equal ["closed", "closed", "closed"], callback_results
1391
+ end
718
1392
  end
719
1393
 
720
1394
  sub_test_case '#server_create_unix' do
@@ -729,6 +1403,22 @@ class ServerPluginHelperTest < Test::Unit::TestCase
729
1403
  # test 'can call close callback if registered'
730
1404
  end
731
1405
 
1406
+ def open_client(proto, addr, port)
1407
+ client = case proto
1408
+ when :tcp
1409
+ TCPSocket.open(addr, port)
1410
+ when :tls
1411
+ c = OpenSSL::SSL::SSLSocket.new(TCPSocket.open(addr, port))
1412
+ c.sync_close = true
1413
+ c.connect
1414
+ else
1415
+ raise ArgumentError, "unknown proto:#{proto}"
1416
+ end
1417
+ yield client
1418
+ ensure
1419
+ client.close rescue nil
1420
+ end
1421
+
732
1422
  # run tests for tcp, tls and unix
733
1423
  sub_test_case '#server_create_connection' do
734
1424
  test 'raise error if udp is specified in proto' do
@@ -737,10 +1427,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
737
1427
  end
738
1428
  end
739
1429
 
740
- # def server_create_connection(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, certopts: nil, resolve_name: false, linger_timeout: 0, backlog: nil, &block)
1430
+ # def server_create_connection(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, tls_options: nil, resolve_name: false, linger_timeout: 0, backlog: nil, &block)
741
1431
  protocols = {
742
1432
  'tcp' => [:tcp, {}],
743
- # 'tls' => [:tls, {certopts: {}}],
1433
+ 'tls' => [:tls, {tls_options: {insecure: true}}],
744
1434
  # 'unix' => [:unix, {path: ""}],
745
1435
  }
746
1436
 
@@ -766,7 +1456,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
766
1456
  end
767
1457
  end
768
1458
  3.times do
769
- TCPSocket.open("127.0.0.1", PORT) do |sock|
1459
+ open_client(proto, "127.0.0.1", PORT) do |sock|
770
1460
  sock.puts "yay"
771
1461
  end
772
1462
  end
@@ -788,7 +1478,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
788
1478
  end
789
1479
  end
790
1480
  3.times do
791
- TCPSocket.open("127.0.0.1", PORT) do |sock|
1481
+ open_client(proto, "127.0.0.1", PORT) do |sock|
792
1482
  sock.puts "yay"
793
1483
  end
794
1484
  end
@@ -816,20 +1506,26 @@ class ServerPluginHelperTest < Test::Unit::TestCase
816
1506
  end
817
1507
  replied = []
818
1508
  disconnecteds = []
819
- 3.times do
820
- TCPSocket.open("127.0.0.1", PORT) do |sock|
1509
+ 3.times do |i|
1510
+ open_client(proto, "127.0.0.1", PORT) do |sock|
821
1511
  sock.puts "yay"
822
1512
  while line = sock.readline
823
1513
  replied << line
824
1514
  break
825
1515
  end
826
1516
  sock.write "x"
1517
+ connection_closed = false
827
1518
  begin
828
- sock.read
1519
+ data = sock.read
1520
+ if data.empty?
1521
+ connection_closed = true
1522
+ end
829
1523
  rescue => e
830
1524
  if e.is_a?(Errno::ECONNRESET)
831
- disconnecteds << e.class
1525
+ connection_closed = true
832
1526
  end
1527
+ ensure
1528
+ disconnecteds << connection_closed
833
1529
  end
834
1530
  end
835
1531
  end
@@ -838,7 +1534,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
838
1534
  waiting(10){ sleep 0.1 until disconnecteds.size == 3 }
839
1535
  assert_equal ["yay\n", "yay\n", "yay\n"], lines
840
1536
  assert_equal ["foo!\n", "foo!\n", "foo!\n"], replied
841
- assert_equal [Errno::ECONNRESET, Errno::ECONNRESET, Errno::ECONNRESET], disconnecteds
1537
+ assert_equal [true, true, true], disconnecteds
842
1538
  end
843
1539
 
844
1540
  data(protocols)
@@ -860,7 +1556,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
860
1556
  end
861
1557
  replied = []
862
1558
  3.times do
863
- TCPSocket.open("127.0.0.1", PORT) do |sock|
1559
+ open_client(proto, "127.0.0.1", PORT) do |sock|
864
1560
  sock.puts "yay"
865
1561
  while line = sock.readline
866
1562
  replied << line
@@ -888,7 +1584,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
888
1584
  end
889
1585
  end
890
1586
  3.times do
891
- TCPSocket.open("127.0.0.1", PORT) do |sock|
1587
+ open_client(proto, "127.0.0.1", PORT) do |sock|
892
1588
  sock.puts "yay"
893
1589
  end
894
1590
  end
@@ -897,6 +1593,82 @@ class ServerPluginHelperTest < Test::Unit::TestCase
897
1593
  assert_equal 0, @d.instance_eval{ @_server_connections.size }
898
1594
  end
899
1595
 
1596
+ data(protocols)
1597
+ test 'will refuse more connect requests after stop, but read data from sockets already connected, in non-shared server' do |(proto, kwargs)|
1598
+ connected = false
1599
+ begin
1600
+ open_client(proto, "127.0.0.1", PORT) do |sock|
1601
+ # expected behavior is connection refused...
1602
+ connected = true
1603
+ end
1604
+ rescue
1605
+ end
1606
+
1607
+ assert_false connected
1608
+
1609
+ received = ""
1610
+ @d.server_create_connection(:s, PORT, proto: proto, shared: false, **kwargs) do |conn|
1611
+ conn.on(:data) do |data|
1612
+ received << data
1613
+ conn.write "ack\n"
1614
+ end
1615
+ end
1616
+
1617
+ th0 = Thread.new do
1618
+ open_client(proto, "127.0.0.1", PORT) do |sock|
1619
+ sock.puts "yay"
1620
+ sock.readline
1621
+ end
1622
+ end
1623
+
1624
+ value0 = waiting(5){ th0.value }
1625
+ assert_equal "ack\n", value0
1626
+
1627
+ stopped = false
1628
+ sleeping = false
1629
+ ending = false
1630
+
1631
+ th1 = Thread.new do
1632
+ open_client(proto, "127.0.0.1", PORT) do |sock|
1633
+ sleeping = true
1634
+ sleep 0.1 until stopped
1635
+ sock.puts "yay"
1636
+ res = sock.readline
1637
+ ending = true
1638
+ res
1639
+ end
1640
+ end
1641
+
1642
+ sleep 0.1 until sleeping
1643
+
1644
+ @d.stop
1645
+ assert @d.stopped?
1646
+ stopped = true
1647
+
1648
+ sleep 0.1 until ending
1649
+
1650
+ @d.before_shutdown
1651
+ @d.shutdown
1652
+
1653
+ th2 = Thread.new do
1654
+ begin
1655
+ open_client(proto, "127.0.0.1", PORT) do |sock|
1656
+ sock.puts "foo"
1657
+ end
1658
+ false # failed
1659
+ rescue
1660
+ true # success
1661
+ end
1662
+ end
1663
+
1664
+ value1 = waiting(5){ th1.value }
1665
+ value2 = waiting(5){ th2.value }
1666
+
1667
+ assert_equal "yay\nyay\n", received
1668
+ assert_equal "ack\n", value1
1669
+ assert value2, "should be truthy value to show connection was correctly refused"
1670
+ end
1671
+
900
1672
  test 'can keep connections alive for tcp/tls if keepalive specified' do
901
1673
  # pend "not implemented yet"
902
1674
  end