fluentd 0.14.11 → 0.14.12
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +1 -5
- data/ChangeLog +54 -2
- data/example/in_dummy_blocks.conf +17 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/logevents.conf +25 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/suppress_config_dump.conf +7 -0
- data/lib/fluent/agent.rb +3 -32
- data/lib/fluent/clock.rb +62 -0
- data/lib/fluent/command/fluentd.rb +12 -0
- data/lib/fluent/compat/input.rb +10 -1
- data/lib/fluent/compat/output.rb +40 -1
- data/lib/fluent/config/configure_proxy.rb +30 -7
- data/lib/fluent/config/section.rb +4 -0
- data/lib/fluent/config/types.rb +2 -2
- data/lib/fluent/configurable.rb +31 -5
- data/lib/fluent/engine.rb +61 -12
- data/lib/fluent/event_router.rb +6 -0
- data/lib/fluent/load.rb +0 -1
- data/lib/fluent/log.rb +118 -42
- data/lib/fluent/match.rb +37 -0
- data/lib/fluent/plugin.rb +25 -3
- data/lib/fluent/plugin/base.rb +4 -0
- data/lib/fluent/plugin/buf_file.rb +38 -14
- data/lib/fluent/plugin/buffer.rb +20 -20
- data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
- data/lib/fluent/plugin/compressable.rb +1 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
- data/lib/fluent/plugin/formatter_csv.rb +4 -1
- data/lib/fluent/plugin/formatter_hash.rb +5 -1
- data/lib/fluent/plugin/formatter_json.rb +10 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
- data/lib/fluent/plugin/in_dummy.rb +4 -0
- data/lib/fluent/plugin/in_exec.rb +4 -0
- data/lib/fluent/plugin/in_forward.rb +11 -3
- data/lib/fluent/plugin/in_gc_stat.rb +4 -0
- data/lib/fluent/plugin/in_http.rb +4 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
- data/lib/fluent/plugin/in_object_space.rb +4 -1
- data/lib/fluent/plugin/in_syslog.rb +4 -0
- data/lib/fluent/plugin/in_tail.rb +193 -116
- data/lib/fluent/plugin/in_tcp.rb +5 -1
- data/lib/fluent/plugin/in_udp.rb +4 -0
- data/lib/fluent/plugin/input.rb +4 -0
- data/lib/fluent/plugin/out_copy.rb +4 -0
- data/lib/fluent/plugin/out_exec.rb +4 -0
- data/lib/fluent/plugin/out_exec_filter.rb +4 -0
- data/lib/fluent/plugin/out_file.rb +70 -30
- data/lib/fluent/plugin/out_forward.rb +132 -28
- data/lib/fluent/plugin/out_null.rb +10 -0
- data/lib/fluent/plugin/out_relabel.rb +4 -0
- data/lib/fluent/plugin/out_roundrobin.rb +4 -0
- data/lib/fluent/plugin/out_secondary_file.rb +5 -0
- data/lib/fluent/plugin/out_stdout.rb +5 -0
- data/lib/fluent/plugin/output.rb +18 -9
- data/lib/fluent/plugin/storage_local.rb +25 -2
- data/lib/fluent/plugin_helper/cert_option.rb +159 -0
- data/lib/fluent/plugin_helper/child_process.rb +6 -6
- data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +29 -4
- data/lib/fluent/plugin_helper/inject.rb +14 -1
- data/lib/fluent/plugin_helper/server.rb +275 -31
- data/lib/fluent/plugin_helper/socket.rb +144 -4
- data/lib/fluent/plugin_helper/socket_option.rb +2 -17
- data/lib/fluent/plugin_helper/storage.rb +7 -1
- data/lib/fluent/plugin_helper/thread.rb +16 -4
- data/lib/fluent/registry.rb +26 -9
- data/lib/fluent/root_agent.rb +7 -3
- data/lib/fluent/supervisor.rb +37 -15
- data/lib/fluent/system_config.rb +37 -10
- data/lib/fluent/test.rb +2 -0
- data/lib/fluent/test/driver/base.rb +24 -26
- data/lib/fluent/test/helpers.rb +21 -0
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +274 -4
- data/test/config/test_configurable.rb +154 -0
- data/test/config/test_configure_proxy.rb +180 -1
- data/test/config/test_system_config.rb +10 -0
- data/test/config/test_types.rb +1 -0
- data/test/plugin/test_base.rb +4 -0
- data/test/plugin/test_buf_file.rb +241 -9
- data/test/plugin/test_buffer.rb +11 -11
- data/test/plugin/test_buffer_file_chunk.rb +6 -6
- data/test/plugin/test_compressable.rb +3 -0
- data/test/plugin/test_filter.rb +4 -0
- data/test/plugin/test_filter_record_transformer.rb +20 -0
- data/test/plugin/test_formatter_csv.rb +9 -0
- data/test/plugin/test_formatter_hash.rb +35 -0
- data/test/plugin/test_formatter_json.rb +8 -0
- data/test/plugin/test_formatter_ltsv.rb +7 -0
- data/test/plugin/test_in_dummy.rb +7 -3
- data/test/plugin/test_in_monitor_agent.rb +43 -5
- data/test/plugin/test_in_tail.rb +97 -4
- data/test/plugin/test_input.rb +4 -0
- data/test/plugin/test_out_file.rb +46 -7
- data/test/plugin/test_out_forward.rb +59 -7
- data/test/plugin/test_output.rb +10 -4
- data/test/plugin/test_output_as_buffered.rb +37 -25
- data/test/plugin/test_output_as_buffered_compress.rb +1 -1
- data/test/plugin/test_output_as_buffered_retries.rb +6 -6
- data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
- data/test/plugin/test_storage_local.rb +40 -1
- data/test/plugin_helper/test_child_process.rb +29 -28
- data/test/plugin_helper/test_compat_parameters.rb +1 -1
- data/test/plugin_helper/test_inject.rb +27 -9
- data/test/plugin_helper/test_server.rb +822 -50
- data/test/plugin_helper/test_storage.rb +11 -0
- data/test/plugin_helper/test_timer.rb +1 -0
- data/test/test_clock.rb +164 -0
- data/test/test_log.rb +146 -15
- data/test/test_plugin.rb +251 -0
- data/test/test_supervisor.rb +65 -57
- data/test/test_test_drivers.rb +2 -2
- metadata +18 -7
- data/lib/fluent/process.rb +0 -504
- data/test/test_process.rb +0 -48
data/test/test_supervisor.rb
CHANGED
@@ -9,10 +9,13 @@ require 'uri'
|
|
9
9
|
require 'fileutils'
|
10
10
|
|
11
11
|
class SupervisorTest < ::Test::Unit::TestCase
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
12
|
+
class DummyServer
|
13
|
+
include Fluent::ServerModule
|
14
|
+
attr_accessor :rpc_endpoint, :enable_get_dump
|
15
|
+
def config
|
16
|
+
{}
|
17
|
+
end
|
18
|
+
end
|
16
19
|
|
17
20
|
TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/supervisor#{ENV['TEST_ENV_NUMBER']}")
|
18
21
|
TMP_ROOT_DIR = File.join(TMP_DIR, 'root')
|
@@ -105,80 +108,83 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
105
108
|
end
|
106
109
|
|
107
110
|
def test_main_process_signal_handlers
|
108
|
-
|
109
|
-
|
110
|
-
unless Fluent.windows?
|
111
|
-
opts = Fluent::Supervisor.default_options
|
112
|
-
sv = Fluent::Supervisor.new(opts)
|
113
|
-
sv.send(:install_main_process_signal_handlers)
|
111
|
+
omit "Windows cannot handle signals" if Fluent.windows?
|
114
112
|
|
115
|
-
|
116
|
-
Process.kill :USR1, $$
|
117
|
-
rescue
|
118
|
-
end
|
113
|
+
create_info_dummy_logger
|
119
114
|
|
120
|
-
|
115
|
+
opts = Fluent::Supervisor.default_options
|
116
|
+
sv = Fluent::Supervisor.new(opts)
|
117
|
+
sv.send(:install_main_process_signal_handlers)
|
121
118
|
|
122
|
-
|
123
|
-
|
119
|
+
begin
|
120
|
+
Process.kill :USR1, $$
|
121
|
+
rescue
|
124
122
|
end
|
125
123
|
|
126
|
-
|
124
|
+
sleep 1
|
125
|
+
|
126
|
+
info_msg = '[info]: force flushing buffered events' + "\n"
|
127
|
+
assert{ $log.out.logs.first.end_with?(info_msg) }
|
128
|
+
ensure
|
129
|
+
$log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
|
127
130
|
end
|
128
131
|
|
129
132
|
def test_supervisor_signal_handler
|
130
|
-
|
131
|
-
|
132
|
-
unless Fluent.windows?
|
133
|
+
omit "Windows cannot handle signals" if Fluent.windows?
|
133
134
|
|
134
|
-
|
135
|
-
begin
|
136
|
-
Process.kill :USR1, $$
|
137
|
-
rescue
|
138
|
-
end
|
139
|
-
|
140
|
-
sleep 1
|
135
|
+
create_debug_dummy_logger
|
141
136
|
|
142
|
-
|
143
|
-
|
137
|
+
server = DummyServer.new
|
138
|
+
server.install_supervisor_signal_handlers
|
139
|
+
begin
|
140
|
+
Process.kill :USR1, $$
|
141
|
+
rescue
|
144
142
|
end
|
145
143
|
|
146
|
-
|
144
|
+
sleep 1
|
145
|
+
|
146
|
+
debug_msg = '[debug]: fluentd supervisor process get SIGUSR1'
|
147
|
+
logs = $log.out.logs
|
148
|
+
assert{ logs.any?{|log| log.include?(debug_msg) } }
|
149
|
+
ensure
|
150
|
+
$log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
|
147
151
|
end
|
148
152
|
|
149
153
|
def test_rpc_server
|
154
|
+
omit "Windows cannot handle signals" if Fluent.windows?
|
155
|
+
|
150
156
|
create_info_dummy_logger
|
151
157
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
conf_data = <<-EOC
|
158
|
+
opts = Fluent::Supervisor.default_options
|
159
|
+
sv = Fluent::Supervisor.new(opts)
|
160
|
+
conf_data = <<-EOC
|
156
161
|
<system>
|
157
162
|
rpc_endpoint 0.0.0.0:24447
|
158
163
|
</system>
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
@rpc_endpoint = sys_conf.rpc_endpoint
|
165
|
-
@enable_get_dump = sys_conf.enable_get_dump
|
164
|
+
EOC
|
165
|
+
conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
|
166
|
+
sv.instance_variable_set(:@conf, conf)
|
167
|
+
sv.send(:set_system_config)
|
168
|
+
sys_conf = sv.instance_variable_get(:@system_config)
|
166
169
|
|
167
|
-
|
170
|
+
server = DummyServer.new
|
171
|
+
server.rpc_endpoint = sys_conf.rpc_endpoint
|
172
|
+
server.enable_get_dump = sys_conf.enable_get_dump
|
168
173
|
|
169
|
-
|
170
|
-
Net::HTTP.get URI.parse('http://0.0.0.0:24447/api/plugins.flushBuffers')
|
171
|
-
info_msg = '[info]: force flushing buffered events' + "\n"
|
174
|
+
server.run_rpc_server
|
172
175
|
|
173
|
-
|
176
|
+
sv.send(:install_main_process_signal_handlers)
|
177
|
+
Net::HTTP.get URI.parse('http://0.0.0.0:24447/api/plugins.flushBuffers')
|
178
|
+
info_msg = '[info]: force flushing buffered events' + "\n"
|
174
179
|
|
175
|
-
|
176
|
-
# This test will be passed in such environment.
|
177
|
-
pend unless $log.out.logs.first
|
180
|
+
server.stop_rpc_server
|
178
181
|
|
179
|
-
|
180
|
-
|
182
|
+
# In TravisCI with OSX(Xcode), it seems that can't use rpc server.
|
183
|
+
# This test will be passed in such environment.
|
184
|
+
pend unless $log.out.logs.first
|
181
185
|
|
186
|
+
assert{ $log.out.logs.first.end_with?(info_msg) }
|
187
|
+
ensure
|
182
188
|
$log.out.reset
|
183
189
|
end
|
184
190
|
|
@@ -197,6 +203,7 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
197
203
|
write_config tmp_dir, conf_info_str
|
198
204
|
|
199
205
|
params = {}
|
206
|
+
params['workers'] = 1
|
200
207
|
params['use_v1_config'] = true
|
201
208
|
params['log_path'] = 'test/tmp/supervisor/log'
|
202
209
|
params['suppress_repeated_stacktrace'] = true
|
@@ -243,7 +250,7 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
243
250
|
|
244
251
|
# fifth call after changed conf file(don't reuse config)
|
245
252
|
se_config = load_config_proc.call
|
246
|
-
assert_equal Fluent::Log::
|
253
|
+
assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
|
247
254
|
end
|
248
255
|
|
249
256
|
def test_load_config_for_daemonize
|
@@ -261,12 +268,13 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
261
268
|
write_config tmp_dir, conf_info_str
|
262
269
|
|
263
270
|
params = {}
|
271
|
+
params['workers'] = 1
|
264
272
|
params['use_v1_config'] = true
|
265
273
|
params['log_path'] = 'test/tmp/supervisor/log'
|
266
274
|
params['suppress_repeated_stacktrace'] = true
|
267
275
|
params['log_level'] = Fluent::Log::LEVEL_INFO
|
268
276
|
params['daemonize'] = './fluentd.pid'
|
269
|
-
load_config_proc =
|
277
|
+
load_config_proc = Proc.new { Fluent::Supervisor.load_config(tmp_dir, params) }
|
270
278
|
|
271
279
|
# first call
|
272
280
|
se_config = load_config_proc.call
|
@@ -308,14 +316,14 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
308
316
|
|
309
317
|
# fifth call after changed conf file(don't reuse config)
|
310
318
|
se_config = load_config_proc.call
|
311
|
-
assert_equal Fluent::Log::
|
319
|
+
assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
|
312
320
|
end
|
313
321
|
|
314
322
|
def test_logger
|
315
323
|
opts = Fluent::Supervisor.default_options
|
316
324
|
sv = Fluent::Supervisor.new(opts)
|
317
325
|
log = sv.instance_variable_get(:@log)
|
318
|
-
log.init
|
326
|
+
log.init(:standalone, 0)
|
319
327
|
logger = $log.instance_variable_get(:@logger)
|
320
328
|
|
321
329
|
assert_equal Fluent::Log::LEVEL_INFO, $log.level
|
@@ -340,7 +348,7 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
340
348
|
)
|
341
349
|
sv = Fluent::Supervisor.new(opts)
|
342
350
|
log = sv.instance_variable_get(:@log)
|
343
|
-
log.init
|
351
|
+
log.init(:standalone, 0)
|
344
352
|
|
345
353
|
assert_equal Fluent::LogDeviceIO, $log.out.class
|
346
354
|
assert_equal rotate_age, $log.out.instance_variable_get(:@shift_age)
|
data/test/test_test_drivers.rb
CHANGED
@@ -67,7 +67,7 @@ class TestDriverTest < ::Test::Unit::TestCase
|
|
67
67
|
assert_nothing_raised do
|
68
68
|
before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
69
69
|
d.end_if{ false }
|
70
|
-
d.run(timeout:
|
70
|
+
d.run(timeout: 5) do
|
71
71
|
sleep 0.1 until d.stop?
|
72
72
|
end
|
73
73
|
after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
@@ -89,7 +89,7 @@ class TestDriverTest < ::Test::Unit::TestCase
|
|
89
89
|
end
|
90
90
|
end
|
91
91
|
end
|
92
|
-
assert_raise "yaaaaaaaaaay!" do
|
92
|
+
assert_raise RuntimeError.new("yaaaaaaaaaay!") do
|
93
93
|
d.end_if{ false }
|
94
94
|
d.run(timeout: 3) do
|
95
95
|
sleep 0.1 until d.stop?
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.14.
|
4
|
+
version: 0.14.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -313,17 +313,21 @@ files:
|
|
313
313
|
- code-of-conduct.md
|
314
314
|
- example/copy_roundrobin.conf
|
315
315
|
- example/filter_stdout.conf
|
316
|
+
- example/in_dummy_blocks.conf
|
316
317
|
- example/in_dummy_with_compression.conf
|
317
318
|
- example/in_forward.conf
|
318
319
|
- example/in_forward_client.conf
|
319
320
|
- example/in_forward_shared_key.conf
|
321
|
+
- example/in_forward_tls.conf
|
320
322
|
- example/in_forward_users.conf
|
323
|
+
- example/in_forward_workers.conf
|
321
324
|
- example/in_http.conf
|
322
325
|
- example/in_out_forward.conf
|
323
326
|
- example/in_syslog.conf
|
324
327
|
- example/in_tail.conf
|
325
328
|
- example/in_tcp.conf
|
326
329
|
- example/in_udp.conf
|
330
|
+
- example/logevents.conf
|
327
331
|
- example/multi_filters.conf
|
328
332
|
- example/out_copy.conf
|
329
333
|
- example/out_exec_filter.conf
|
@@ -331,15 +335,19 @@ files:
|
|
331
335
|
- example/out_forward.conf
|
332
336
|
- example/out_forward_buf_file.conf
|
333
337
|
- example/out_forward_client.conf
|
338
|
+
- example/out_forward_heartbeat_none.conf
|
334
339
|
- example/out_forward_shared_key.conf
|
340
|
+
- example/out_forward_tls.conf
|
335
341
|
- example/out_forward_users.conf
|
336
342
|
- example/out_null.conf
|
337
343
|
- example/secondary_file.conf
|
344
|
+
- example/suppress_config_dump.conf
|
338
345
|
- example/v0_12_filter.conf
|
339
346
|
- example/v1_literal_example.conf
|
340
347
|
- fluent.conf
|
341
348
|
- fluentd.gemspec
|
342
349
|
- lib/fluent/agent.rb
|
350
|
+
- lib/fluent/clock.rb
|
343
351
|
- lib/fluent/command/binlog_reader.rb
|
344
352
|
- lib/fluent/command/bundler_injection.rb
|
345
353
|
- lib/fluent/command/cat.rb
|
@@ -471,6 +479,7 @@ files:
|
|
471
479
|
- lib/fluent/plugin/storage_local.rb
|
472
480
|
- lib/fluent/plugin/string_util.rb
|
473
481
|
- lib/fluent/plugin_helper.rb
|
482
|
+
- lib/fluent/plugin_helper/cert_option.rb
|
474
483
|
- lib/fluent/plugin_helper/child_process.rb
|
475
484
|
- lib/fluent/plugin_helper/compat_parameters.rb
|
476
485
|
- lib/fluent/plugin_helper/event_emitter.rb
|
@@ -487,7 +496,6 @@ files:
|
|
487
496
|
- lib/fluent/plugin_helper/thread.rb
|
488
497
|
- lib/fluent/plugin_helper/timer.rb
|
489
498
|
- lib/fluent/plugin_id.rb
|
490
|
-
- lib/fluent/process.rb
|
491
499
|
- lib/fluent/registry.rb
|
492
500
|
- lib/fluent/root_agent.rb
|
493
501
|
- lib/fluent/rpc.rb
|
@@ -556,6 +564,7 @@ files:
|
|
556
564
|
- test/plugin/test_filter_record_transformer.rb
|
557
565
|
- test/plugin/test_filter_stdout.rb
|
558
566
|
- test/plugin/test_formatter_csv.rb
|
567
|
+
- test/plugin/test_formatter_hash.rb
|
559
568
|
- test/plugin/test_formatter_json.rb
|
560
569
|
- test/plugin/test_formatter_ltsv.rb
|
561
570
|
- test/plugin/test_formatter_msgpack.rb
|
@@ -631,6 +640,7 @@ files:
|
|
631
640
|
- test/scripts/fluent/plugin/out_test.rb
|
632
641
|
- test/scripts/fluent/plugin/out_test2.rb
|
633
642
|
- test/scripts/fluent/plugin/parser_known.rb
|
643
|
+
- test/test_clock.rb
|
634
644
|
- test/test_config.rb
|
635
645
|
- test/test_configdsl.rb
|
636
646
|
- test/test_event.rb
|
@@ -643,10 +653,10 @@ files:
|
|
643
653
|
- test/test_match.rb
|
644
654
|
- test/test_mixin.rb
|
645
655
|
- test/test_output.rb
|
656
|
+
- test/test_plugin.rb
|
646
657
|
- test/test_plugin_classes.rb
|
647
658
|
- test/test_plugin_helper.rb
|
648
659
|
- test/test_plugin_id.rb
|
649
|
-
- test/test_process.rb
|
650
660
|
- test/test_root_agent.rb
|
651
661
|
- test/test_supervisor.rb
|
652
662
|
- test/test_test_drivers.rb
|
@@ -673,7 +683,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
673
683
|
version: '0'
|
674
684
|
requirements: []
|
675
685
|
rubyforge_project:
|
676
|
-
rubygems_version: 2.
|
686
|
+
rubygems_version: 2.6.8
|
677
687
|
signing_key:
|
678
688
|
specification_version: 4
|
679
689
|
summary: Fluentd event collector
|
@@ -715,6 +725,7 @@ test_files:
|
|
715
725
|
- test/plugin/test_filter_record_transformer.rb
|
716
726
|
- test/plugin/test_filter_stdout.rb
|
717
727
|
- test/plugin/test_formatter_csv.rb
|
728
|
+
- test/plugin/test_formatter_hash.rb
|
718
729
|
- test/plugin/test_formatter_json.rb
|
719
730
|
- test/plugin/test_formatter_ltsv.rb
|
720
731
|
- test/plugin/test_formatter_msgpack.rb
|
@@ -790,6 +801,7 @@ test_files:
|
|
790
801
|
- test/scripts/fluent/plugin/out_test.rb
|
791
802
|
- test/scripts/fluent/plugin/out_test2.rb
|
792
803
|
- test/scripts/fluent/plugin/parser_known.rb
|
804
|
+
- test/test_clock.rb
|
793
805
|
- test/test_config.rb
|
794
806
|
- test/test_configdsl.rb
|
795
807
|
- test/test_event.rb
|
@@ -802,14 +814,13 @@ test_files:
|
|
802
814
|
- test/test_match.rb
|
803
815
|
- test/test_mixin.rb
|
804
816
|
- test/test_output.rb
|
817
|
+
- test/test_plugin.rb
|
805
818
|
- test/test_plugin_classes.rb
|
806
819
|
- test/test_plugin_helper.rb
|
807
820
|
- test/test_plugin_id.rb
|
808
|
-
- test/test_process.rb
|
809
821
|
- test/test_root_agent.rb
|
810
822
|
- test/test_supervisor.rb
|
811
823
|
- test/test_test_drivers.rb
|
812
824
|
- test/test_time_formatter.rb
|
813
825
|
- test/test_time_parser.rb
|
814
826
|
- test/test_unique_id.rb
|
815
|
-
has_rdoc: false
|
data/lib/fluent/process.rb
DELETED
@@ -1,504 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Fluentd
|
3
|
-
#
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
-
# you may not use this file except in compliance with the License.
|
6
|
-
# You may obtain a copy of the License at
|
7
|
-
#
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
-
#
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
-
# See the License for the specific language governing permissions and
|
14
|
-
# limitations under the License.
|
15
|
-
#
|
16
|
-
|
17
|
-
# This feature will be deprecated after introducing
|
18
|
-
# symmetric multi processing in core.
|
19
|
-
|
20
|
-
require 'thread'
|
21
|
-
|
22
|
-
require 'fluent/config'
|
23
|
-
## This comment out (to remove circular reference) is a bit risky,
|
24
|
-
## but in all known cases this file is required after 'fluent/engine'.
|
25
|
-
# require 'fluent/engine'
|
26
|
-
require 'fluent/event'
|
27
|
-
|
28
|
-
module Fluent
|
29
|
-
class DetachProcessManager
|
30
|
-
require 'singleton'
|
31
|
-
include Singleton
|
32
|
-
|
33
|
-
class Broker
|
34
|
-
def initialize
|
35
|
-
end
|
36
|
-
|
37
|
-
def engine
|
38
|
-
Engine
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def initialize
|
43
|
-
require 'drb'
|
44
|
-
DRb.start_service(create_drb_uri, Broker.new)
|
45
|
-
@parent_uri = DRb.uri
|
46
|
-
end
|
47
|
-
|
48
|
-
def fork(delegate_object)
|
49
|
-
ipr, ipw = IO.pipe # child Engine.emit_stream -> parent Engine.emit_stream
|
50
|
-
opr, opw = IO.pipe # parent target.emit_events -> child target.emit_events
|
51
|
-
|
52
|
-
pid = Process.fork
|
53
|
-
if pid
|
54
|
-
# parent process
|
55
|
-
ipw.close
|
56
|
-
opr.close
|
57
|
-
forward_thread = process_parent(ipr, opw, pid, delegate_object)
|
58
|
-
return pid, forward_thread
|
59
|
-
end
|
60
|
-
|
61
|
-
# child process
|
62
|
-
ipr.close
|
63
|
-
opw.close
|
64
|
-
forward_thread = process_child(ipw, opr, delegate_object)
|
65
|
-
return nil, forward_thread
|
66
|
-
end
|
67
|
-
|
68
|
-
private
|
69
|
-
def read_header(ipr)
|
70
|
-
sz = ipr.read(4).unpack('N')[0]
|
71
|
-
ipr.read(sz)
|
72
|
-
end
|
73
|
-
|
74
|
-
def send_header(ipw, data)
|
75
|
-
ipw.write [data.bytesize].pack('N')
|
76
|
-
ipw.write data
|
77
|
-
ipw.flush
|
78
|
-
end
|
79
|
-
|
80
|
-
def create_drb_uri
|
81
|
-
"drbunix:" # TODO
|
82
|
-
end
|
83
|
-
|
84
|
-
private
|
85
|
-
def process_child(ipw, opr, delegate_object)
|
86
|
-
DRb.start_service(create_drb_uri, delegate_object)
|
87
|
-
child_uri = DRb.uri
|
88
|
-
|
89
|
-
send_header(ipw, child_uri)
|
90
|
-
|
91
|
-
# override target.emit_stream to write event stream to the pipe
|
92
|
-
fwd = new_forwarder(ipw, 0.5) # TODO interval
|
93
|
-
Engine.define_singleton_method(:emit_stream) do |tag,es|
|
94
|
-
fwd.emit(tag, es)
|
95
|
-
end
|
96
|
-
|
97
|
-
# read event stream from the pipe and forward to target.emit
|
98
|
-
forward_thread = Thread.new(opr, delegate_object, &method(:output_forward_main))
|
99
|
-
|
100
|
-
# override global methods to call parent process
|
101
|
-
override_shared_methods(@parent_uri)
|
102
|
-
|
103
|
-
return forward_thread
|
104
|
-
end
|
105
|
-
|
106
|
-
def override_shared_methods(parent_uri)
|
107
|
-
broker = DRbObject.new_with_uri(parent_uri)
|
108
|
-
shared_methods.each {|(broker_accessor,target,name)|
|
109
|
-
remote = broker.send(broker_accessor)
|
110
|
-
target.define_singleton_method(name) do |*args,&block|
|
111
|
-
remote.send(name, *args, &block)
|
112
|
-
end
|
113
|
-
}
|
114
|
-
end
|
115
|
-
|
116
|
-
def shared_methods
|
117
|
-
[
|
118
|
-
#[:engine, Engine, :flush!],
|
119
|
-
#[:engine, Engine, :stop],
|
120
|
-
]
|
121
|
-
end
|
122
|
-
|
123
|
-
def process_parent(ipr, opw, pid, delegate_object)
|
124
|
-
# child_uri = read_header(ipr)
|
125
|
-
|
126
|
-
# read event stream from the pipe and forward to Engine.emit_stream
|
127
|
-
forward_thread = Thread.new(ipr, pid, &method(:input_forward_main))
|
128
|
-
|
129
|
-
# note: don't override methods in parent process
|
130
|
-
# because another process may fork after overriding
|
131
|
-
#override_delegate_methods(delegate_object, child_uri)
|
132
|
-
|
133
|
-
# return forwarder for DetachProcessMixin to
|
134
|
-
# override target.emit and write event stream to the pipe
|
135
|
-
fwd = new_forwarder(opw, 0.5) # TODO interval
|
136
|
-
# note: override emit method on DetachProcessMixin
|
137
|
-
forward_thread.define_singleton_method(:forwarder) do
|
138
|
-
fwd
|
139
|
-
end
|
140
|
-
|
141
|
-
return forward_thread
|
142
|
-
end
|
143
|
-
|
144
|
-
#def override_delegate_methods(target, child_uri)
|
145
|
-
# remote = DRbObject.new_with_uri(child_uri)
|
146
|
-
# delegate_methods(target).each {|name|
|
147
|
-
# target.define_singleton_method(name) do |*args,&block|
|
148
|
-
# remote.send(name, *args, &block)
|
149
|
-
# end
|
150
|
-
# }
|
151
|
-
#end
|
152
|
-
#
|
153
|
-
#def delegate_methods(target)
|
154
|
-
# target.methods - Object.public_instance_methods
|
155
|
-
#end
|
156
|
-
|
157
|
-
def output_forward_main(opr, target)
|
158
|
-
read_event_stream(opr) {|tag,es|
|
159
|
-
# FIXME error handling
|
160
|
-
begin
|
161
|
-
target.emit_events(tag, es)
|
162
|
-
rescue
|
163
|
-
$log.warn "failed to emit", error: $!.to_s, pid: Process.pid
|
164
|
-
$log.warn_backtrace
|
165
|
-
end
|
166
|
-
}
|
167
|
-
rescue
|
168
|
-
$log.error "error on output process forwarding thread", error: $!.to_s, pid: Process.pid
|
169
|
-
$log.error_backtrace
|
170
|
-
raise
|
171
|
-
end
|
172
|
-
|
173
|
-
def input_forward_main(ipr, pid)
|
174
|
-
read_event_stream(ipr) {|tag,es|
|
175
|
-
# FIXME error handling
|
176
|
-
begin
|
177
|
-
Engine.emit_stream(tag, es)
|
178
|
-
rescue
|
179
|
-
$log.warn "failed to emit", error: $!.to_s, pid: Process.pid
|
180
|
-
$log.warn_backtrace
|
181
|
-
end
|
182
|
-
}
|
183
|
-
rescue
|
184
|
-
$log.error "error on input process forwarding thread", error: $!.to_s, pid: Process.pid
|
185
|
-
$log.error_backtrace
|
186
|
-
raise
|
187
|
-
end
|
188
|
-
|
189
|
-
def read_event_stream(r, &block)
|
190
|
-
u = Fluent::Engine.msgpack_factory.unpacker(r)
|
191
|
-
begin
|
192
|
-
#buf = ''
|
193
|
-
#map = {}
|
194
|
-
#while true
|
195
|
-
# r.readpartial(64*1024, buf)
|
196
|
-
# u.feed_each(buf) {|tag,ms|
|
197
|
-
# if msbuf = map[tag]
|
198
|
-
# msbuf << ms
|
199
|
-
# else
|
200
|
-
# map[tag] = ms
|
201
|
-
# end
|
202
|
-
# }
|
203
|
-
# unless map.empty?
|
204
|
-
# map.each_pair {|tag,ms|
|
205
|
-
# es = MessagePackEventStream.new(ms)
|
206
|
-
# block.call(tag, es)
|
207
|
-
# }
|
208
|
-
# map.clear
|
209
|
-
# end
|
210
|
-
#end
|
211
|
-
u.each {|tag,ms|
|
212
|
-
es = MessagePackEventStream.new(ms)
|
213
|
-
block.call(tag, es)
|
214
|
-
}
|
215
|
-
rescue EOFError
|
216
|
-
ensure
|
217
|
-
r.close
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
def new_forwarder(w, interval)
|
222
|
-
if interval < 0.2 # TODO interval
|
223
|
-
Forwarder.new(w)
|
224
|
-
else
|
225
|
-
DelayedForwarder.new(w, interval)
|
226
|
-
end
|
227
|
-
end
|
228
|
-
|
229
|
-
class Forwarder
|
230
|
-
def initialize(w)
|
231
|
-
@w = w
|
232
|
-
end
|
233
|
-
|
234
|
-
def emit(tag, es)
|
235
|
-
ms = es.to_msgpack_stream
|
236
|
-
#[tag, ms].to_msgpack(@w) # not thread safe
|
237
|
-
@w.write [tag, ms].to_msgpack
|
238
|
-
end
|
239
|
-
end
|
240
|
-
|
241
|
-
class DelayedForwarder
|
242
|
-
def initialize(w, interval)
|
243
|
-
@w = w
|
244
|
-
@interval = interval
|
245
|
-
@buffer = {}
|
246
|
-
@mutex = Mutex.new
|
247
|
-
Thread.new(&method(:run))
|
248
|
-
end
|
249
|
-
|
250
|
-
def emit(tag, es)
|
251
|
-
stream = es.to_msgpack_stream
|
252
|
-
@mutex.synchronize do
|
253
|
-
if @buffer[tag]
|
254
|
-
@buffer[tag] << stream
|
255
|
-
else
|
256
|
-
@buffer[tag] = stream
|
257
|
-
end
|
258
|
-
end
|
259
|
-
end
|
260
|
-
|
261
|
-
def run
|
262
|
-
while true
|
263
|
-
sleep @interval
|
264
|
-
|
265
|
-
pairs = []
|
266
|
-
@mutex.synchronize do
|
267
|
-
@buffer.keys.each do |tag|
|
268
|
-
if ms = @buffer.delete(tag)
|
269
|
-
pairs << [tag, ms]
|
270
|
-
end
|
271
|
-
end
|
272
|
-
end
|
273
|
-
pairs.each do |pair|
|
274
|
-
pair.to_msgpack(@w)
|
275
|
-
end
|
276
|
-
end
|
277
|
-
rescue
|
278
|
-
$log.error "error on forwerder thread", error: $!.to_s
|
279
|
-
$log.error_backtrace
|
280
|
-
raise
|
281
|
-
end
|
282
|
-
end
|
283
|
-
|
284
|
-
class MultiForwarder
|
285
|
-
def initialize(forwarders)
|
286
|
-
@forwarders = forwarders
|
287
|
-
@rr = 1
|
288
|
-
end
|
289
|
-
|
290
|
-
def emit(tag, es)
|
291
|
-
forwarder = @forwarders[@rr]
|
292
|
-
@rr = (@rr + 1) % @forwarders.length
|
293
|
-
forwarder.emit(tag, es)
|
294
|
-
end
|
295
|
-
end
|
296
|
-
end
|
297
|
-
|
298
|
-
|
299
|
-
module DetachProcessImpl
|
300
|
-
def on_detach_process(i)
|
301
|
-
end
|
302
|
-
|
303
|
-
def on_exit_process(i)
|
304
|
-
end
|
305
|
-
|
306
|
-
private
|
307
|
-
|
308
|
-
def detach_process_impl(num, &block)
|
309
|
-
children = []
|
310
|
-
|
311
|
-
num.times do |i|
|
312
|
-
pid, forward_thread = DetachProcessManager.instance.fork(self)
|
313
|
-
|
314
|
-
if pid
|
315
|
-
# parent process
|
316
|
-
$log.info "detached process", class: self.class, pid: pid
|
317
|
-
children << [pid, forward_thread]
|
318
|
-
next
|
319
|
-
end
|
320
|
-
|
321
|
-
# child process
|
322
|
-
begin
|
323
|
-
on_detach_process(i)
|
324
|
-
|
325
|
-
block.call
|
326
|
-
|
327
|
-
# disable Engine.stop called by signal handler
|
328
|
-
Engine.define_singleton_method(:stop) do
|
329
|
-
# do nothing
|
330
|
-
end
|
331
|
-
|
332
|
-
# override signal handlers called by parent process
|
333
|
-
fin = FinishWait.new
|
334
|
-
trap :INT do
|
335
|
-
fin.stop
|
336
|
-
end
|
337
|
-
trap :TERM do
|
338
|
-
fin.stop
|
339
|
-
end
|
340
|
-
#forward_thread.join # TODO this thread won't stop because parent doesn't close pipe
|
341
|
-
fin.wait
|
342
|
-
|
343
|
-
on_exit_process(i)
|
344
|
-
exit! 0
|
345
|
-
ensure
|
346
|
-
$log.error "unknown error while shutting down this child process", error: $!.to_s, pid: Process.pid
|
347
|
-
$log.error_backtrace
|
348
|
-
end
|
349
|
-
|
350
|
-
exit! 1
|
351
|
-
end
|
352
|
-
|
353
|
-
# parent process
|
354
|
-
# override shutdown method to kill child processes
|
355
|
-
define_singleton_method(:shutdown) do
|
356
|
-
children.each {|pair|
|
357
|
-
begin
|
358
|
-
pid = pair[0]
|
359
|
-
forward_thread = pair[1]
|
360
|
-
if pid
|
361
|
-
Process.kill(:TERM, pid)
|
362
|
-
forward_thread.join # wait until child closes pipe
|
363
|
-
Process.waitpid(pid)
|
364
|
-
pair[0] = nil
|
365
|
-
end
|
366
|
-
rescue
|
367
|
-
$log.error "unknown error while shutting down remote child process", error: $!.to_s
|
368
|
-
$log.error_backtrace
|
369
|
-
end
|
370
|
-
}
|
371
|
-
end
|
372
|
-
|
373
|
-
# override target.emit_events and write event stream to the pipe
|
374
|
-
forwarders = children.map {|pair| pair[1].forwarder }
|
375
|
-
if forwarders.length > 1
|
376
|
-
# use roundrobin
|
377
|
-
fwd = DetachProcessManager::MultiForwarder.new(forwarders)
|
378
|
-
else
|
379
|
-
fwd = forwarders[0]
|
380
|
-
end
|
381
|
-
define_singleton_method(:emit_events) do |tag,es|
|
382
|
-
fwd.emit_events(tag, es)
|
383
|
-
end
|
384
|
-
end
|
385
|
-
|
386
|
-
class FinishWait
|
387
|
-
def initialize
|
388
|
-
@finished = false
|
389
|
-
@mutex = Mutex.new
|
390
|
-
@cond = ConditionVariable.new
|
391
|
-
end
|
392
|
-
|
393
|
-
def wait
|
394
|
-
@mutex.synchronize do
|
395
|
-
until @finished
|
396
|
-
@cond.wait(@mutex, 1.0)
|
397
|
-
end
|
398
|
-
end
|
399
|
-
end
|
400
|
-
|
401
|
-
def stop
|
402
|
-
return if @finished
|
403
|
-
@finished = true
|
404
|
-
# Creating new thread due to mutex can't lock in main thread during trap context
|
405
|
-
Thread.new {
|
406
|
-
@mutex.synchronize do
|
407
|
-
@cond.broadcast
|
408
|
-
end
|
409
|
-
}.run
|
410
|
-
end
|
411
|
-
|
412
|
-
def finished?
|
413
|
-
@finished
|
414
|
-
end
|
415
|
-
end
|
416
|
-
end
|
417
|
-
|
418
|
-
|
419
|
-
module DetachProcessMixin
|
420
|
-
include DetachProcessImpl
|
421
|
-
|
422
|
-
def configure(conf)
|
423
|
-
super
|
424
|
-
|
425
|
-
@detach_process = nil
|
426
|
-
|
427
|
-
if detach_process = conf['detach_process']
|
428
|
-
b3v = Config.bool_value(detach_process)
|
429
|
-
case b3v
|
430
|
-
when nil
|
431
|
-
num = detach_process.to_i
|
432
|
-
if num > 1
|
433
|
-
$log.warn "'detach_process' parameter supports only 1 process on this plugin: #{conf}"
|
434
|
-
elsif num > 0
|
435
|
-
@detach_process = true
|
436
|
-
elsif detach_process =~ /0+/
|
437
|
-
@detach_process = false
|
438
|
-
else
|
439
|
-
@detach_process = true
|
440
|
-
end
|
441
|
-
when true
|
442
|
-
@detach_process = true
|
443
|
-
when false
|
444
|
-
@detach_process = false
|
445
|
-
end
|
446
|
-
end
|
447
|
-
end
|
448
|
-
|
449
|
-
def detach_process(&block)
|
450
|
-
if @detach_process
|
451
|
-
detach_process_impl(1, &block)
|
452
|
-
else
|
453
|
-
block.call
|
454
|
-
end
|
455
|
-
end
|
456
|
-
end
|
457
|
-
|
458
|
-
|
459
|
-
module DetachMultiProcessMixin
|
460
|
-
include DetachProcessImpl
|
461
|
-
|
462
|
-
def initialize
|
463
|
-
@detach_process_num = 2
|
464
|
-
super
|
465
|
-
end
|
466
|
-
|
467
|
-
def configure(conf)
|
468
|
-
super
|
469
|
-
|
470
|
-
@detach_process = nil
|
471
|
-
|
472
|
-
if detach_process = conf['detach_process']
|
473
|
-
b3v = Config.bool_value(detach_process)
|
474
|
-
case b3v
|
475
|
-
when nil
|
476
|
-
num = detach_process.to_i
|
477
|
-
if num > 0
|
478
|
-
@detach_process = true
|
479
|
-
@detach_process_num = num
|
480
|
-
elsif detach_process =~ /0+/
|
481
|
-
@detach_process = false
|
482
|
-
else
|
483
|
-
@detach_process = true
|
484
|
-
end
|
485
|
-
when true
|
486
|
-
@detach_process = true
|
487
|
-
when false
|
488
|
-
@detach_process = false
|
489
|
-
end
|
490
|
-
end
|
491
|
-
end
|
492
|
-
|
493
|
-
private
|
494
|
-
|
495
|
-
def detach_multi_process(&block)
|
496
|
-
if @detach_process
|
497
|
-
detach_process_impl(@detach_process_num, &block)
|
498
|
-
else
|
499
|
-
block.call
|
500
|
-
end
|
501
|
-
end
|
502
|
-
end
|
503
|
-
end
|
504
|
-
|