fluentd 0.14.11-x86-mingw32 → 0.14.12-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -5
  3. data/ChangeLog +54 -2
  4. data/example/in_dummy_blocks.conf +17 -0
  5. data/example/in_forward_tls.conf +14 -0
  6. data/example/in_forward_workers.conf +21 -0
  7. data/example/logevents.conf +25 -0
  8. data/example/out_forward_heartbeat_none.conf +16 -0
  9. data/example/out_forward_tls.conf +18 -0
  10. data/example/suppress_config_dump.conf +7 -0
  11. data/lib/fluent/agent.rb +3 -32
  12. data/lib/fluent/clock.rb +62 -0
  13. data/lib/fluent/command/fluentd.rb +12 -0
  14. data/lib/fluent/compat/input.rb +10 -1
  15. data/lib/fluent/compat/output.rb +40 -1
  16. data/lib/fluent/config/configure_proxy.rb +30 -7
  17. data/lib/fluent/config/section.rb +4 -0
  18. data/lib/fluent/config/types.rb +2 -2
  19. data/lib/fluent/configurable.rb +31 -5
  20. data/lib/fluent/engine.rb +61 -12
  21. data/lib/fluent/event_router.rb +6 -0
  22. data/lib/fluent/load.rb +0 -1
  23. data/lib/fluent/log.rb +118 -42
  24. data/lib/fluent/match.rb +37 -0
  25. data/lib/fluent/plugin.rb +25 -3
  26. data/lib/fluent/plugin/base.rb +4 -0
  27. data/lib/fluent/plugin/buf_file.rb +38 -14
  28. data/lib/fluent/plugin/buffer.rb +20 -20
  29. data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
  30. data/lib/fluent/plugin/compressable.rb +1 -0
  31. data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
  32. data/lib/fluent/plugin/formatter_csv.rb +4 -1
  33. data/lib/fluent/plugin/formatter_hash.rb +5 -1
  34. data/lib/fluent/plugin/formatter_json.rb +10 -0
  35. data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
  36. data/lib/fluent/plugin/in_dummy.rb +4 -0
  37. data/lib/fluent/plugin/in_exec.rb +4 -0
  38. data/lib/fluent/plugin/in_forward.rb +11 -3
  39. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  40. data/lib/fluent/plugin/in_http.rb +4 -0
  41. data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
  42. data/lib/fluent/plugin/in_object_space.rb +4 -1
  43. data/lib/fluent/plugin/in_syslog.rb +4 -0
  44. data/lib/fluent/plugin/in_tail.rb +193 -116
  45. data/lib/fluent/plugin/in_tcp.rb +5 -1
  46. data/lib/fluent/plugin/in_udp.rb +4 -0
  47. data/lib/fluent/plugin/input.rb +4 -0
  48. data/lib/fluent/plugin/out_copy.rb +4 -0
  49. data/lib/fluent/plugin/out_exec.rb +4 -0
  50. data/lib/fluent/plugin/out_exec_filter.rb +4 -0
  51. data/lib/fluent/plugin/out_file.rb +70 -30
  52. data/lib/fluent/plugin/out_forward.rb +132 -28
  53. data/lib/fluent/plugin/out_null.rb +10 -0
  54. data/lib/fluent/plugin/out_relabel.rb +4 -0
  55. data/lib/fluent/plugin/out_roundrobin.rb +4 -0
  56. data/lib/fluent/plugin/out_secondary_file.rb +5 -0
  57. data/lib/fluent/plugin/out_stdout.rb +5 -0
  58. data/lib/fluent/plugin/output.rb +18 -9
  59. data/lib/fluent/plugin/storage_local.rb +25 -2
  60. data/lib/fluent/plugin_helper/cert_option.rb +159 -0
  61. data/lib/fluent/plugin_helper/child_process.rb +6 -6
  62. data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
  63. data/lib/fluent/plugin_helper/event_loop.rb +29 -4
  64. data/lib/fluent/plugin_helper/inject.rb +14 -1
  65. data/lib/fluent/plugin_helper/server.rb +275 -31
  66. data/lib/fluent/plugin_helper/socket.rb +144 -4
  67. data/lib/fluent/plugin_helper/socket_option.rb +2 -17
  68. data/lib/fluent/plugin_helper/storage.rb +7 -1
  69. data/lib/fluent/plugin_helper/thread.rb +16 -4
  70. data/lib/fluent/registry.rb +26 -9
  71. data/lib/fluent/root_agent.rb +7 -3
  72. data/lib/fluent/supervisor.rb +37 -15
  73. data/lib/fluent/system_config.rb +37 -10
  74. data/lib/fluent/test.rb +2 -0
  75. data/lib/fluent/test/driver/base.rb +24 -26
  76. data/lib/fluent/test/helpers.rb +21 -0
  77. data/lib/fluent/version.rb +1 -1
  78. data/test/command/test_fluentd.rb +274 -4
  79. data/test/config/test_configurable.rb +154 -0
  80. data/test/config/test_configure_proxy.rb +180 -1
  81. data/test/config/test_system_config.rb +10 -0
  82. data/test/config/test_types.rb +1 -0
  83. data/test/plugin/test_base.rb +4 -0
  84. data/test/plugin/test_buf_file.rb +241 -9
  85. data/test/plugin/test_buffer.rb +11 -11
  86. data/test/plugin/test_buffer_file_chunk.rb +6 -6
  87. data/test/plugin/test_compressable.rb +3 -0
  88. data/test/plugin/test_filter.rb +4 -0
  89. data/test/plugin/test_filter_record_transformer.rb +20 -0
  90. data/test/plugin/test_formatter_csv.rb +9 -0
  91. data/test/plugin/test_formatter_hash.rb +35 -0
  92. data/test/plugin/test_formatter_json.rb +8 -0
  93. data/test/plugin/test_formatter_ltsv.rb +7 -0
  94. data/test/plugin/test_in_dummy.rb +7 -3
  95. data/test/plugin/test_in_monitor_agent.rb +43 -5
  96. data/test/plugin/test_in_tail.rb +97 -4
  97. data/test/plugin/test_input.rb +4 -0
  98. data/test/plugin/test_out_file.rb +46 -7
  99. data/test/plugin/test_out_forward.rb +59 -7
  100. data/test/plugin/test_output.rb +10 -4
  101. data/test/plugin/test_output_as_buffered.rb +37 -25
  102. data/test/plugin/test_output_as_buffered_compress.rb +1 -1
  103. data/test/plugin/test_output_as_buffered_retries.rb +6 -6
  104. data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
  105. data/test/plugin/test_storage_local.rb +40 -1
  106. data/test/plugin_helper/test_child_process.rb +29 -28
  107. data/test/plugin_helper/test_compat_parameters.rb +1 -1
  108. data/test/plugin_helper/test_inject.rb +27 -9
  109. data/test/plugin_helper/test_server.rb +822 -50
  110. data/test/plugin_helper/test_storage.rb +11 -0
  111. data/test/plugin_helper/test_timer.rb +1 -0
  112. data/test/test_clock.rb +164 -0
  113. data/test/test_log.rb +146 -15
  114. data/test/test_plugin.rb +251 -0
  115. data/test/test_supervisor.rb +65 -57
  116. data/test/test_test_drivers.rb +2 -2
  117. metadata +18 -7
  118. data/lib/fluent/process.rb +0 -504
  119. data/test/test_process.rb +0 -48
@@ -9,10 +9,13 @@ require 'uri'
9
9
  require 'fileutils'
10
10
 
11
11
  class SupervisorTest < ::Test::Unit::TestCase
12
- include Fluent
13
- include FluentTest
14
- include ServerModule
15
- include WorkerModule
12
+ class DummyServer
13
+ include Fluent::ServerModule
14
+ attr_accessor :rpc_endpoint, :enable_get_dump
15
+ def config
16
+ {}
17
+ end
18
+ end
16
19
 
17
20
  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/supervisor#{ENV['TEST_ENV_NUMBER']}")
18
21
  TMP_ROOT_DIR = File.join(TMP_DIR, 'root')
@@ -105,80 +108,83 @@ class SupervisorTest < ::Test::Unit::TestCase
105
108
  end
106
109
 
107
110
  def test_main_process_signal_handlers
108
- create_info_dummy_logger
109
-
110
- unless Fluent.windows?
111
- opts = Fluent::Supervisor.default_options
112
- sv = Fluent::Supervisor.new(opts)
113
- sv.send(:install_main_process_signal_handlers)
111
+ omit "Windows cannot handle signals" if Fluent.windows?
114
112
 
115
- begin
116
- Process.kill :USR1, $$
117
- rescue
118
- end
113
+ create_info_dummy_logger
119
114
 
120
- sleep 1
115
+ opts = Fluent::Supervisor.default_options
116
+ sv = Fluent::Supervisor.new(opts)
117
+ sv.send(:install_main_process_signal_handlers)
121
118
 
122
- info_msg = '[info]: force flushing buffered events' + "\n"
123
- assert{ $log.out.logs.first.end_with?(info_msg) }
119
+ begin
120
+ Process.kill :USR1, $$
121
+ rescue
124
122
  end
125
123
 
126
- $log.out.reset
124
+ sleep 1
125
+
126
+ info_msg = '[info]: force flushing buffered events' + "\n"
127
+ assert{ $log.out.logs.first.end_with?(info_msg) }
128
+ ensure
129
+ $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
127
130
  end
128
131
 
129
132
  def test_supervisor_signal_handler
130
- create_debug_dummy_logger
131
-
132
- unless Fluent.windows?
133
+ omit "Windows cannot handle signals" if Fluent.windows?
133
134
 
134
- install_supervisor_signal_handlers
135
- begin
136
- Process.kill :USR1, $$
137
- rescue
138
- end
139
-
140
- sleep 1
135
+ create_debug_dummy_logger
141
136
 
142
- debug_msg = '[debug]: fluentd supervisor process get SIGUSR1' + "\n"
143
- assert{ $log.out.logs.first.end_with?(debug_msg) }
137
+ server = DummyServer.new
138
+ server.install_supervisor_signal_handlers
139
+ begin
140
+ Process.kill :USR1, $$
141
+ rescue
144
142
  end
145
143
 
146
- $log.out.reset
144
+ sleep 1
145
+
146
+ debug_msg = '[debug]: fluentd supervisor process get SIGUSR1'
147
+ logs = $log.out.logs
148
+ assert{ logs.any?{|log| log.include?(debug_msg) } }
149
+ ensure
150
+ $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
147
151
  end
148
152
 
149
153
  def test_rpc_server
154
+ omit "Windows cannot handle signals" if Fluent.windows?
155
+
150
156
  create_info_dummy_logger
151
157
 
152
- unless Fluent.windows?
153
- opts = Fluent::Supervisor.default_options
154
- sv = Fluent::Supervisor.new(opts)
155
- conf_data = <<-EOC
158
+ opts = Fluent::Supervisor.default_options
159
+ sv = Fluent::Supervisor.new(opts)
160
+ conf_data = <<-EOC
156
161
  <system>
157
162
  rpc_endpoint 0.0.0.0:24447
158
163
  </system>
159
- EOC
160
- conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
161
- sv.instance_variable_set(:@conf, conf)
162
- sv.send(:set_system_config)
163
- sys_conf = sv.instance_variable_get(:@system_config)
164
- @rpc_endpoint = sys_conf.rpc_endpoint
165
- @enable_get_dump = sys_conf.enable_get_dump
164
+ EOC
165
+ conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
166
+ sv.instance_variable_set(:@conf, conf)
167
+ sv.send(:set_system_config)
168
+ sys_conf = sv.instance_variable_get(:@system_config)
166
169
 
167
- run_rpc_server
170
+ server = DummyServer.new
171
+ server.rpc_endpoint = sys_conf.rpc_endpoint
172
+ server.enable_get_dump = sys_conf.enable_get_dump
168
173
 
169
- sv.send(:install_main_process_signal_handlers)
170
- Net::HTTP.get URI.parse('http://0.0.0.0:24447/api/plugins.flushBuffers')
171
- info_msg = '[info]: force flushing buffered events' + "\n"
174
+ server.run_rpc_server
172
175
 
173
- stop_rpc_server
176
+ sv.send(:install_main_process_signal_handlers)
177
+ Net::HTTP.get URI.parse('http://0.0.0.0:24447/api/plugins.flushBuffers')
178
+ info_msg = '[info]: force flushing buffered events' + "\n"
174
179
 
175
- # In TravisCI with OSX(Xcode), it seems that can't use rpc server.
176
- # This test will be passed in such environment.
177
- pend unless $log.out.logs.first
180
+ server.stop_rpc_server
178
181
 
179
- assert{ $log.out.logs.first.end_with?(info_msg) }
180
- end
182
+ # In TravisCI with OSX(Xcode), it seems that can't use rpc server.
183
+ # This test will be passed in such environment.
184
+ pend unless $log.out.logs.first
181
185
 
186
+ assert{ $log.out.logs.first.end_with?(info_msg) }
187
+ ensure
182
188
  $log.out.reset
183
189
  end
184
190
 
@@ -197,6 +203,7 @@ class SupervisorTest < ::Test::Unit::TestCase
197
203
  write_config tmp_dir, conf_info_str
198
204
 
199
205
  params = {}
206
+ params['workers'] = 1
200
207
  params['use_v1_config'] = true
201
208
  params['log_path'] = 'test/tmp/supervisor/log'
202
209
  params['suppress_repeated_stacktrace'] = true
@@ -243,7 +250,7 @@ class SupervisorTest < ::Test::Unit::TestCase
243
250
 
244
251
  # fifth call after changed conf file(don't reuse config)
245
252
  se_config = load_config_proc.call
246
- assert_equal Fluent::Log::LEVEL_DEBUG, se_config[:log_level]
253
+ assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
247
254
  end
248
255
 
249
256
  def test_load_config_for_daemonize
@@ -261,12 +268,13 @@ class SupervisorTest < ::Test::Unit::TestCase
261
268
  write_config tmp_dir, conf_info_str
262
269
 
263
270
  params = {}
271
+ params['workers'] = 1
264
272
  params['use_v1_config'] = true
265
273
  params['log_path'] = 'test/tmp/supervisor/log'
266
274
  params['suppress_repeated_stacktrace'] = true
267
275
  params['log_level'] = Fluent::Log::LEVEL_INFO
268
276
  params['daemonize'] = './fluentd.pid'
269
- load_config_proc = Proc.new { Fluent::Supervisor.load_config(tmp_dir, params) }
277
+ load_config_proc = Proc.new { Fluent::Supervisor.load_config(tmp_dir, params) }
270
278
 
271
279
  # first call
272
280
  se_config = load_config_proc.call
@@ -308,14 +316,14 @@ class SupervisorTest < ::Test::Unit::TestCase
308
316
 
309
317
  # fifth call after changed conf file(don't reuse config)
310
318
  se_config = load_config_proc.call
311
- assert_equal Fluent::Log::LEVEL_DEBUG, se_config[:log_level]
319
+ assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
312
320
  end
313
321
 
314
322
  def test_logger
315
323
  opts = Fluent::Supervisor.default_options
316
324
  sv = Fluent::Supervisor.new(opts)
317
325
  log = sv.instance_variable_get(:@log)
318
- log.init
326
+ log.init(:standalone, 0)
319
327
  logger = $log.instance_variable_get(:@logger)
320
328
 
321
329
  assert_equal Fluent::Log::LEVEL_INFO, $log.level
@@ -340,7 +348,7 @@ class SupervisorTest < ::Test::Unit::TestCase
340
348
  )
341
349
  sv = Fluent::Supervisor.new(opts)
342
350
  log = sv.instance_variable_get(:@log)
343
- log.init
351
+ log.init(:standalone, 0)
344
352
 
345
353
  assert_equal Fluent::LogDeviceIO, $log.out.class
346
354
  assert_equal rotate_age, $log.out.instance_variable_get(:@shift_age)
@@ -67,7 +67,7 @@ class TestDriverTest < ::Test::Unit::TestCase
67
67
  assert_nothing_raised do
68
68
  before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
69
69
  d.end_if{ false }
70
- d.run(timeout: 1) do
70
+ d.run(timeout: 5) do
71
71
  sleep 0.1 until d.stop?
72
72
  end
73
73
  after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -89,7 +89,7 @@ class TestDriverTest < ::Test::Unit::TestCase
89
89
  end
90
90
  end
91
91
  end
92
- assert_raise "yaaaaaaaaaay!" do
92
+ assert_raise RuntimeError.new("yaaaaaaaaaay!") do
93
93
  d.end_if{ false }
94
94
  d.run(timeout: 3) do
95
95
  sleep 0.1 until d.stop?
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.11
4
+ version: 0.14.12
5
5
  platform: x86-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-26 00:00:00.000000000 Z
11
+ date: 2017-01-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -369,17 +369,21 @@ files:
369
369
  - code-of-conduct.md
370
370
  - example/copy_roundrobin.conf
371
371
  - example/filter_stdout.conf
372
+ - example/in_dummy_blocks.conf
372
373
  - example/in_dummy_with_compression.conf
373
374
  - example/in_forward.conf
374
375
  - example/in_forward_client.conf
375
376
  - example/in_forward_shared_key.conf
377
+ - example/in_forward_tls.conf
376
378
  - example/in_forward_users.conf
379
+ - example/in_forward_workers.conf
377
380
  - example/in_http.conf
378
381
  - example/in_out_forward.conf
379
382
  - example/in_syslog.conf
380
383
  - example/in_tail.conf
381
384
  - example/in_tcp.conf
382
385
  - example/in_udp.conf
386
+ - example/logevents.conf
383
387
  - example/multi_filters.conf
384
388
  - example/out_copy.conf
385
389
  - example/out_exec_filter.conf
@@ -387,15 +391,19 @@ files:
387
391
  - example/out_forward.conf
388
392
  - example/out_forward_buf_file.conf
389
393
  - example/out_forward_client.conf
394
+ - example/out_forward_heartbeat_none.conf
390
395
  - example/out_forward_shared_key.conf
396
+ - example/out_forward_tls.conf
391
397
  - example/out_forward_users.conf
392
398
  - example/out_null.conf
393
399
  - example/secondary_file.conf
400
+ - example/suppress_config_dump.conf
394
401
  - example/v0_12_filter.conf
395
402
  - example/v1_literal_example.conf
396
403
  - fluent.conf
397
404
  - fluentd.gemspec
398
405
  - lib/fluent/agent.rb
406
+ - lib/fluent/clock.rb
399
407
  - lib/fluent/command/binlog_reader.rb
400
408
  - lib/fluent/command/bundler_injection.rb
401
409
  - lib/fluent/command/cat.rb
@@ -527,6 +535,7 @@ files:
527
535
  - lib/fluent/plugin/storage_local.rb
528
536
  - lib/fluent/plugin/string_util.rb
529
537
  - lib/fluent/plugin_helper.rb
538
+ - lib/fluent/plugin_helper/cert_option.rb
530
539
  - lib/fluent/plugin_helper/child_process.rb
531
540
  - lib/fluent/plugin_helper/compat_parameters.rb
532
541
  - lib/fluent/plugin_helper/event_emitter.rb
@@ -543,7 +552,6 @@ files:
543
552
  - lib/fluent/plugin_helper/thread.rb
544
553
  - lib/fluent/plugin_helper/timer.rb
545
554
  - lib/fluent/plugin_id.rb
546
- - lib/fluent/process.rb
547
555
  - lib/fluent/registry.rb
548
556
  - lib/fluent/root_agent.rb
549
557
  - lib/fluent/rpc.rb
@@ -612,6 +620,7 @@ files:
612
620
  - test/plugin/test_filter_record_transformer.rb
613
621
  - test/plugin/test_filter_stdout.rb
614
622
  - test/plugin/test_formatter_csv.rb
623
+ - test/plugin/test_formatter_hash.rb
615
624
  - test/plugin/test_formatter_json.rb
616
625
  - test/plugin/test_formatter_ltsv.rb
617
626
  - test/plugin/test_formatter_msgpack.rb
@@ -687,6 +696,7 @@ files:
687
696
  - test/scripts/fluent/plugin/out_test.rb
688
697
  - test/scripts/fluent/plugin/out_test2.rb
689
698
  - test/scripts/fluent/plugin/parser_known.rb
699
+ - test/test_clock.rb
690
700
  - test/test_config.rb
691
701
  - test/test_configdsl.rb
692
702
  - test/test_event.rb
@@ -699,10 +709,10 @@ files:
699
709
  - test/test_match.rb
700
710
  - test/test_mixin.rb
701
711
  - test/test_output.rb
712
+ - test/test_plugin.rb
702
713
  - test/test_plugin_classes.rb
703
714
  - test/test_plugin_helper.rb
704
715
  - test/test_plugin_id.rb
705
- - test/test_process.rb
706
716
  - test/test_root_agent.rb
707
717
  - test/test_supervisor.rb
708
718
  - test/test_test_drivers.rb
@@ -729,7 +739,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
729
739
  version: '0'
730
740
  requirements: []
731
741
  rubyforge_project:
732
- rubygems_version: 2.5.1
742
+ rubygems_version: 2.6.8
733
743
  signing_key:
734
744
  specification_version: 4
735
745
  summary: Fluentd event collector
@@ -771,6 +781,7 @@ test_files:
771
781
  - test/plugin/test_filter_record_transformer.rb
772
782
  - test/plugin/test_filter_stdout.rb
773
783
  - test/plugin/test_formatter_csv.rb
784
+ - test/plugin/test_formatter_hash.rb
774
785
  - test/plugin/test_formatter_json.rb
775
786
  - test/plugin/test_formatter_ltsv.rb
776
787
  - test/plugin/test_formatter_msgpack.rb
@@ -846,6 +857,7 @@ test_files:
846
857
  - test/scripts/fluent/plugin/out_test.rb
847
858
  - test/scripts/fluent/plugin/out_test2.rb
848
859
  - test/scripts/fluent/plugin/parser_known.rb
860
+ - test/test_clock.rb
849
861
  - test/test_config.rb
850
862
  - test/test_configdsl.rb
851
863
  - test/test_event.rb
@@ -858,14 +870,13 @@ test_files:
858
870
  - test/test_match.rb
859
871
  - test/test_mixin.rb
860
872
  - test/test_output.rb
873
+ - test/test_plugin.rb
861
874
  - test/test_plugin_classes.rb
862
875
  - test/test_plugin_helper.rb
863
876
  - test/test_plugin_id.rb
864
- - test/test_process.rb
865
877
  - test/test_root_agent.rb
866
878
  - test/test_supervisor.rb
867
879
  - test/test_test_drivers.rb
868
880
  - test/test_time_formatter.rb
869
881
  - test/test_time_parser.rb
870
882
  - test/test_unique_id.rb
871
- has_rdoc: false
@@ -1,504 +0,0 @@
1
- #
2
- # Fluentd
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
- #
16
-
17
- # This feature will be deprecated after introducing
18
- # symmetric multi processing in core.
19
-
20
- require 'thread'
21
-
22
- require 'fluent/config'
23
- ## This comment out (to remove circular reference) is a bit risky,
24
- ## but in all known cases this file is required after 'fluent/engine'.
25
- # require 'fluent/engine'
26
- require 'fluent/event'
27
-
28
- module Fluent
29
- class DetachProcessManager
30
- require 'singleton'
31
- include Singleton
32
-
33
- class Broker
34
- def initialize
35
- end
36
-
37
- def engine
38
- Engine
39
- end
40
- end
41
-
42
- def initialize
43
- require 'drb'
44
- DRb.start_service(create_drb_uri, Broker.new)
45
- @parent_uri = DRb.uri
46
- end
47
-
48
- def fork(delegate_object)
49
- ipr, ipw = IO.pipe # child Engine.emit_stream -> parent Engine.emit_stream
50
- opr, opw = IO.pipe # parent target.emit_events -> child target.emit_events
51
-
52
- pid = Process.fork
53
- if pid
54
- # parent process
55
- ipw.close
56
- opr.close
57
- forward_thread = process_parent(ipr, opw, pid, delegate_object)
58
- return pid, forward_thread
59
- end
60
-
61
- # child process
62
- ipr.close
63
- opw.close
64
- forward_thread = process_child(ipw, opr, delegate_object)
65
- return nil, forward_thread
66
- end
67
-
68
- private
69
- def read_header(ipr)
70
- sz = ipr.read(4).unpack('N')[0]
71
- ipr.read(sz)
72
- end
73
-
74
- def send_header(ipw, data)
75
- ipw.write [data.bytesize].pack('N')
76
- ipw.write data
77
- ipw.flush
78
- end
79
-
80
- def create_drb_uri
81
- "drbunix:" # TODO
82
- end
83
-
84
- private
85
- def process_child(ipw, opr, delegate_object)
86
- DRb.start_service(create_drb_uri, delegate_object)
87
- child_uri = DRb.uri
88
-
89
- send_header(ipw, child_uri)
90
-
91
- # override target.emit_stream to write event stream to the pipe
92
- fwd = new_forwarder(ipw, 0.5) # TODO interval
93
- Engine.define_singleton_method(:emit_stream) do |tag,es|
94
- fwd.emit(tag, es)
95
- end
96
-
97
- # read event stream from the pipe and forward to target.emit
98
- forward_thread = Thread.new(opr, delegate_object, &method(:output_forward_main))
99
-
100
- # override global methods to call parent process
101
- override_shared_methods(@parent_uri)
102
-
103
- return forward_thread
104
- end
105
-
106
- def override_shared_methods(parent_uri)
107
- broker = DRbObject.new_with_uri(parent_uri)
108
- shared_methods.each {|(broker_accessor,target,name)|
109
- remote = broker.send(broker_accessor)
110
- target.define_singleton_method(name) do |*args,&block|
111
- remote.send(name, *args, &block)
112
- end
113
- }
114
- end
115
-
116
- def shared_methods
117
- [
118
- #[:engine, Engine, :flush!],
119
- #[:engine, Engine, :stop],
120
- ]
121
- end
122
-
123
- def process_parent(ipr, opw, pid, delegate_object)
124
- # child_uri = read_header(ipr)
125
-
126
- # read event stream from the pipe and forward to Engine.emit_stream
127
- forward_thread = Thread.new(ipr, pid, &method(:input_forward_main))
128
-
129
- # note: don't override methods in parent process
130
- # because another process may fork after overriding
131
- #override_delegate_methods(delegate_object, child_uri)
132
-
133
- # return forwarder for DetachProcessMixin to
134
- # override target.emit and write event stream to the pipe
135
- fwd = new_forwarder(opw, 0.5) # TODO interval
136
- # note: override emit method on DetachProcessMixin
137
- forward_thread.define_singleton_method(:forwarder) do
138
- fwd
139
- end
140
-
141
- return forward_thread
142
- end
143
-
144
- #def override_delegate_methods(target, child_uri)
145
- # remote = DRbObject.new_with_uri(child_uri)
146
- # delegate_methods(target).each {|name|
147
- # target.define_singleton_method(name) do |*args,&block|
148
- # remote.send(name, *args, &block)
149
- # end
150
- # }
151
- #end
152
- #
153
- #def delegate_methods(target)
154
- # target.methods - Object.public_instance_methods
155
- #end
156
-
157
- def output_forward_main(opr, target)
158
- read_event_stream(opr) {|tag,es|
159
- # FIXME error handling
160
- begin
161
- target.emit_events(tag, es)
162
- rescue
163
- $log.warn "failed to emit", error: $!.to_s, pid: Process.pid
164
- $log.warn_backtrace
165
- end
166
- }
167
- rescue
168
- $log.error "error on output process forwarding thread", error: $!.to_s, pid: Process.pid
169
- $log.error_backtrace
170
- raise
171
- end
172
-
173
- def input_forward_main(ipr, pid)
174
- read_event_stream(ipr) {|tag,es|
175
- # FIXME error handling
176
- begin
177
- Engine.emit_stream(tag, es)
178
- rescue
179
- $log.warn "failed to emit", error: $!.to_s, pid: Process.pid
180
- $log.warn_backtrace
181
- end
182
- }
183
- rescue
184
- $log.error "error on input process forwarding thread", error: $!.to_s, pid: Process.pid
185
- $log.error_backtrace
186
- raise
187
- end
188
-
189
- def read_event_stream(r, &block)
190
- u = Fluent::Engine.msgpack_factory.unpacker(r)
191
- begin
192
- #buf = ''
193
- #map = {}
194
- #while true
195
- # r.readpartial(64*1024, buf)
196
- # u.feed_each(buf) {|tag,ms|
197
- # if msbuf = map[tag]
198
- # msbuf << ms
199
- # else
200
- # map[tag] = ms
201
- # end
202
- # }
203
- # unless map.empty?
204
- # map.each_pair {|tag,ms|
205
- # es = MessagePackEventStream.new(ms)
206
- # block.call(tag, es)
207
- # }
208
- # map.clear
209
- # end
210
- #end
211
- u.each {|tag,ms|
212
- es = MessagePackEventStream.new(ms)
213
- block.call(tag, es)
214
- }
215
- rescue EOFError
216
- ensure
217
- r.close
218
- end
219
- end
220
-
221
- def new_forwarder(w, interval)
222
- if interval < 0.2 # TODO interval
223
- Forwarder.new(w)
224
- else
225
- DelayedForwarder.new(w, interval)
226
- end
227
- end
228
-
229
- class Forwarder
230
- def initialize(w)
231
- @w = w
232
- end
233
-
234
- def emit(tag, es)
235
- ms = es.to_msgpack_stream
236
- #[tag, ms].to_msgpack(@w) # not thread safe
237
- @w.write [tag, ms].to_msgpack
238
- end
239
- end
240
-
241
- class DelayedForwarder
242
- def initialize(w, interval)
243
- @w = w
244
- @interval = interval
245
- @buffer = {}
246
- @mutex = Mutex.new
247
- Thread.new(&method(:run))
248
- end
249
-
250
- def emit(tag, es)
251
- stream = es.to_msgpack_stream
252
- @mutex.synchronize do
253
- if @buffer[tag]
254
- @buffer[tag] << stream
255
- else
256
- @buffer[tag] = stream
257
- end
258
- end
259
- end
260
-
261
- def run
262
- while true
263
- sleep @interval
264
-
265
- pairs = []
266
- @mutex.synchronize do
267
- @buffer.keys.each do |tag|
268
- if ms = @buffer.delete(tag)
269
- pairs << [tag, ms]
270
- end
271
- end
272
- end
273
- pairs.each do |pair|
274
- pair.to_msgpack(@w)
275
- end
276
- end
277
- rescue
278
- $log.error "error on forwerder thread", error: $!.to_s
279
- $log.error_backtrace
280
- raise
281
- end
282
- end
283
-
284
- class MultiForwarder
285
- def initialize(forwarders)
286
- @forwarders = forwarders
287
- @rr = 1
288
- end
289
-
290
- def emit(tag, es)
291
- forwarder = @forwarders[@rr]
292
- @rr = (@rr + 1) % @forwarders.length
293
- forwarder.emit(tag, es)
294
- end
295
- end
296
- end
297
-
298
-
299
- module DetachProcessImpl
300
- def on_detach_process(i)
301
- end
302
-
303
- def on_exit_process(i)
304
- end
305
-
306
- private
307
-
308
- def detach_process_impl(num, &block)
309
- children = []
310
-
311
- num.times do |i|
312
- pid, forward_thread = DetachProcessManager.instance.fork(self)
313
-
314
- if pid
315
- # parent process
316
- $log.info "detached process", class: self.class, pid: pid
317
- children << [pid, forward_thread]
318
- next
319
- end
320
-
321
- # child process
322
- begin
323
- on_detach_process(i)
324
-
325
- block.call
326
-
327
- # disable Engine.stop called by signal handler
328
- Engine.define_singleton_method(:stop) do
329
- # do nothing
330
- end
331
-
332
- # override signal handlers called by parent process
333
- fin = FinishWait.new
334
- trap :INT do
335
- fin.stop
336
- end
337
- trap :TERM do
338
- fin.stop
339
- end
340
- #forward_thread.join # TODO this thread won't stop because parent doesn't close pipe
341
- fin.wait
342
-
343
- on_exit_process(i)
344
- exit! 0
345
- ensure
346
- $log.error "unknown error while shutting down this child process", error: $!.to_s, pid: Process.pid
347
- $log.error_backtrace
348
- end
349
-
350
- exit! 1
351
- end
352
-
353
- # parent process
354
- # override shutdown method to kill child processes
355
- define_singleton_method(:shutdown) do
356
- children.each {|pair|
357
- begin
358
- pid = pair[0]
359
- forward_thread = pair[1]
360
- if pid
361
- Process.kill(:TERM, pid)
362
- forward_thread.join # wait until child closes pipe
363
- Process.waitpid(pid)
364
- pair[0] = nil
365
- end
366
- rescue
367
- $log.error "unknown error while shutting down remote child process", error: $!.to_s
368
- $log.error_backtrace
369
- end
370
- }
371
- end
372
-
373
- # override target.emit_events and write event stream to the pipe
374
- forwarders = children.map {|pair| pair[1].forwarder }
375
- if forwarders.length > 1
376
- # use roundrobin
377
- fwd = DetachProcessManager::MultiForwarder.new(forwarders)
378
- else
379
- fwd = forwarders[0]
380
- end
381
- define_singleton_method(:emit_events) do |tag,es|
382
- fwd.emit_events(tag, es)
383
- end
384
- end
385
-
386
- class FinishWait
387
- def initialize
388
- @finished = false
389
- @mutex = Mutex.new
390
- @cond = ConditionVariable.new
391
- end
392
-
393
- def wait
394
- @mutex.synchronize do
395
- until @finished
396
- @cond.wait(@mutex, 1.0)
397
- end
398
- end
399
- end
400
-
401
- def stop
402
- return if @finished
403
- @finished = true
404
- # Creating new thread due to mutex can't lock in main thread during trap context
405
- Thread.new {
406
- @mutex.synchronize do
407
- @cond.broadcast
408
- end
409
- }.run
410
- end
411
-
412
- def finished?
413
- @finished
414
- end
415
- end
416
- end
417
-
418
-
419
- module DetachProcessMixin
420
- include DetachProcessImpl
421
-
422
- def configure(conf)
423
- super
424
-
425
- @detach_process = nil
426
-
427
- if detach_process = conf['detach_process']
428
- b3v = Config.bool_value(detach_process)
429
- case b3v
430
- when nil
431
- num = detach_process.to_i
432
- if num > 1
433
- $log.warn "'detach_process' parameter supports only 1 process on this plugin: #{conf}"
434
- elsif num > 0
435
- @detach_process = true
436
- elsif detach_process =~ /0+/
437
- @detach_process = false
438
- else
439
- @detach_process = true
440
- end
441
- when true
442
- @detach_process = true
443
- when false
444
- @detach_process = false
445
- end
446
- end
447
- end
448
-
449
- def detach_process(&block)
450
- if @detach_process
451
- detach_process_impl(1, &block)
452
- else
453
- block.call
454
- end
455
- end
456
- end
457
-
458
-
459
- module DetachMultiProcessMixin
460
- include DetachProcessImpl
461
-
462
- def initialize
463
- @detach_process_num = 2
464
- super
465
- end
466
-
467
- def configure(conf)
468
- super
469
-
470
- @detach_process = nil
471
-
472
- if detach_process = conf['detach_process']
473
- b3v = Config.bool_value(detach_process)
474
- case b3v
475
- when nil
476
- num = detach_process.to_i
477
- if num > 0
478
- @detach_process = true
479
- @detach_process_num = num
480
- elsif detach_process =~ /0+/
481
- @detach_process = false
482
- else
483
- @detach_process = true
484
- end
485
- when true
486
- @detach_process = true
487
- when false
488
- @detach_process = false
489
- end
490
- end
491
- end
492
-
493
- private
494
-
495
- def detach_multi_process(&block)
496
- if @detach_process
497
- detach_process_impl(@detach_process_num, &block)
498
- else
499
- block.call
500
- end
501
- end
502
- end
503
- end
504
-