fluentd 0.14.11-x86-mingw32 → 0.14.12-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +1 -5
- data/ChangeLog +54 -2
- data/example/in_dummy_blocks.conf +17 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/logevents.conf +25 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/suppress_config_dump.conf +7 -0
- data/lib/fluent/agent.rb +3 -32
- data/lib/fluent/clock.rb +62 -0
- data/lib/fluent/command/fluentd.rb +12 -0
- data/lib/fluent/compat/input.rb +10 -1
- data/lib/fluent/compat/output.rb +40 -1
- data/lib/fluent/config/configure_proxy.rb +30 -7
- data/lib/fluent/config/section.rb +4 -0
- data/lib/fluent/config/types.rb +2 -2
- data/lib/fluent/configurable.rb +31 -5
- data/lib/fluent/engine.rb +61 -12
- data/lib/fluent/event_router.rb +6 -0
- data/lib/fluent/load.rb +0 -1
- data/lib/fluent/log.rb +118 -42
- data/lib/fluent/match.rb +37 -0
- data/lib/fluent/plugin.rb +25 -3
- data/lib/fluent/plugin/base.rb +4 -0
- data/lib/fluent/plugin/buf_file.rb +38 -14
- data/lib/fluent/plugin/buffer.rb +20 -20
- data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
- data/lib/fluent/plugin/compressable.rb +1 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
- data/lib/fluent/plugin/formatter_csv.rb +4 -1
- data/lib/fluent/plugin/formatter_hash.rb +5 -1
- data/lib/fluent/plugin/formatter_json.rb +10 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
- data/lib/fluent/plugin/in_dummy.rb +4 -0
- data/lib/fluent/plugin/in_exec.rb +4 -0
- data/lib/fluent/plugin/in_forward.rb +11 -3
- data/lib/fluent/plugin/in_gc_stat.rb +4 -0
- data/lib/fluent/plugin/in_http.rb +4 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
- data/lib/fluent/plugin/in_object_space.rb +4 -1
- data/lib/fluent/plugin/in_syslog.rb +4 -0
- data/lib/fluent/plugin/in_tail.rb +193 -116
- data/lib/fluent/plugin/in_tcp.rb +5 -1
- data/lib/fluent/plugin/in_udp.rb +4 -0
- data/lib/fluent/plugin/input.rb +4 -0
- data/lib/fluent/plugin/out_copy.rb +4 -0
- data/lib/fluent/plugin/out_exec.rb +4 -0
- data/lib/fluent/plugin/out_exec_filter.rb +4 -0
- data/lib/fluent/plugin/out_file.rb +70 -30
- data/lib/fluent/plugin/out_forward.rb +132 -28
- data/lib/fluent/plugin/out_null.rb +10 -0
- data/lib/fluent/plugin/out_relabel.rb +4 -0
- data/lib/fluent/plugin/out_roundrobin.rb +4 -0
- data/lib/fluent/plugin/out_secondary_file.rb +5 -0
- data/lib/fluent/plugin/out_stdout.rb +5 -0
- data/lib/fluent/plugin/output.rb +18 -9
- data/lib/fluent/plugin/storage_local.rb +25 -2
- data/lib/fluent/plugin_helper/cert_option.rb +159 -0
- data/lib/fluent/plugin_helper/child_process.rb +6 -6
- data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +29 -4
- data/lib/fluent/plugin_helper/inject.rb +14 -1
- data/lib/fluent/plugin_helper/server.rb +275 -31
- data/lib/fluent/plugin_helper/socket.rb +144 -4
- data/lib/fluent/plugin_helper/socket_option.rb +2 -17
- data/lib/fluent/plugin_helper/storage.rb +7 -1
- data/lib/fluent/plugin_helper/thread.rb +16 -4
- data/lib/fluent/registry.rb +26 -9
- data/lib/fluent/root_agent.rb +7 -3
- data/lib/fluent/supervisor.rb +37 -15
- data/lib/fluent/system_config.rb +37 -10
- data/lib/fluent/test.rb +2 -0
- data/lib/fluent/test/driver/base.rb +24 -26
- data/lib/fluent/test/helpers.rb +21 -0
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +274 -4
- data/test/config/test_configurable.rb +154 -0
- data/test/config/test_configure_proxy.rb +180 -1
- data/test/config/test_system_config.rb +10 -0
- data/test/config/test_types.rb +1 -0
- data/test/plugin/test_base.rb +4 -0
- data/test/plugin/test_buf_file.rb +241 -9
- data/test/plugin/test_buffer.rb +11 -11
- data/test/plugin/test_buffer_file_chunk.rb +6 -6
- data/test/plugin/test_compressable.rb +3 -0
- data/test/plugin/test_filter.rb +4 -0
- data/test/plugin/test_filter_record_transformer.rb +20 -0
- data/test/plugin/test_formatter_csv.rb +9 -0
- data/test/plugin/test_formatter_hash.rb +35 -0
- data/test/plugin/test_formatter_json.rb +8 -0
- data/test/plugin/test_formatter_ltsv.rb +7 -0
- data/test/plugin/test_in_dummy.rb +7 -3
- data/test/plugin/test_in_monitor_agent.rb +43 -5
- data/test/plugin/test_in_tail.rb +97 -4
- data/test/plugin/test_input.rb +4 -0
- data/test/plugin/test_out_file.rb +46 -7
- data/test/plugin/test_out_forward.rb +59 -7
- data/test/plugin/test_output.rb +10 -4
- data/test/plugin/test_output_as_buffered.rb +37 -25
- data/test/plugin/test_output_as_buffered_compress.rb +1 -1
- data/test/plugin/test_output_as_buffered_retries.rb +6 -6
- data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
- data/test/plugin/test_storage_local.rb +40 -1
- data/test/plugin_helper/test_child_process.rb +29 -28
- data/test/plugin_helper/test_compat_parameters.rb +1 -1
- data/test/plugin_helper/test_inject.rb +27 -9
- data/test/plugin_helper/test_server.rb +822 -50
- data/test/plugin_helper/test_storage.rb +11 -0
- data/test/plugin_helper/test_timer.rb +1 -0
- data/test/test_clock.rb +164 -0
- data/test/test_log.rb +146 -15
- data/test/test_plugin.rb +251 -0
- data/test/test_supervisor.rb +65 -57
- data/test/test_test_drivers.rb +2 -2
- metadata +18 -7
- data/lib/fluent/process.rb +0 -504
- data/test/test_process.rb +0 -48
data/test/test_supervisor.rb
CHANGED
@@ -9,10 +9,13 @@ require 'uri'
|
|
9
9
|
require 'fileutils'
|
10
10
|
|
11
11
|
class SupervisorTest < ::Test::Unit::TestCase
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
12
|
+
class DummyServer
|
13
|
+
include Fluent::ServerModule
|
14
|
+
attr_accessor :rpc_endpoint, :enable_get_dump
|
15
|
+
def config
|
16
|
+
{}
|
17
|
+
end
|
18
|
+
end
|
16
19
|
|
17
20
|
TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/supervisor#{ENV['TEST_ENV_NUMBER']}")
|
18
21
|
TMP_ROOT_DIR = File.join(TMP_DIR, 'root')
|
@@ -105,80 +108,83 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
105
108
|
end
|
106
109
|
|
107
110
|
def test_main_process_signal_handlers
|
108
|
-
|
109
|
-
|
110
|
-
unless Fluent.windows?
|
111
|
-
opts = Fluent::Supervisor.default_options
|
112
|
-
sv = Fluent::Supervisor.new(opts)
|
113
|
-
sv.send(:install_main_process_signal_handlers)
|
111
|
+
omit "Windows cannot handle signals" if Fluent.windows?
|
114
112
|
|
115
|
-
|
116
|
-
Process.kill :USR1, $$
|
117
|
-
rescue
|
118
|
-
end
|
113
|
+
create_info_dummy_logger
|
119
114
|
|
120
|
-
|
115
|
+
opts = Fluent::Supervisor.default_options
|
116
|
+
sv = Fluent::Supervisor.new(opts)
|
117
|
+
sv.send(:install_main_process_signal_handlers)
|
121
118
|
|
122
|
-
|
123
|
-
|
119
|
+
begin
|
120
|
+
Process.kill :USR1, $$
|
121
|
+
rescue
|
124
122
|
end
|
125
123
|
|
126
|
-
|
124
|
+
sleep 1
|
125
|
+
|
126
|
+
info_msg = '[info]: force flushing buffered events' + "\n"
|
127
|
+
assert{ $log.out.logs.first.end_with?(info_msg) }
|
128
|
+
ensure
|
129
|
+
$log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
|
127
130
|
end
|
128
131
|
|
129
132
|
def test_supervisor_signal_handler
|
130
|
-
|
131
|
-
|
132
|
-
unless Fluent.windows?
|
133
|
+
omit "Windows cannot handle signals" if Fluent.windows?
|
133
134
|
|
134
|
-
|
135
|
-
begin
|
136
|
-
Process.kill :USR1, $$
|
137
|
-
rescue
|
138
|
-
end
|
139
|
-
|
140
|
-
sleep 1
|
135
|
+
create_debug_dummy_logger
|
141
136
|
|
142
|
-
|
143
|
-
|
137
|
+
server = DummyServer.new
|
138
|
+
server.install_supervisor_signal_handlers
|
139
|
+
begin
|
140
|
+
Process.kill :USR1, $$
|
141
|
+
rescue
|
144
142
|
end
|
145
143
|
|
146
|
-
|
144
|
+
sleep 1
|
145
|
+
|
146
|
+
debug_msg = '[debug]: fluentd supervisor process get SIGUSR1'
|
147
|
+
logs = $log.out.logs
|
148
|
+
assert{ logs.any?{|log| log.include?(debug_msg) } }
|
149
|
+
ensure
|
150
|
+
$log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
|
147
151
|
end
|
148
152
|
|
149
153
|
def test_rpc_server
|
154
|
+
omit "Windows cannot handle signals" if Fluent.windows?
|
155
|
+
|
150
156
|
create_info_dummy_logger
|
151
157
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
conf_data = <<-EOC
|
158
|
+
opts = Fluent::Supervisor.default_options
|
159
|
+
sv = Fluent::Supervisor.new(opts)
|
160
|
+
conf_data = <<-EOC
|
156
161
|
<system>
|
157
162
|
rpc_endpoint 0.0.0.0:24447
|
158
163
|
</system>
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
@rpc_endpoint = sys_conf.rpc_endpoint
|
165
|
-
@enable_get_dump = sys_conf.enable_get_dump
|
164
|
+
EOC
|
165
|
+
conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
|
166
|
+
sv.instance_variable_set(:@conf, conf)
|
167
|
+
sv.send(:set_system_config)
|
168
|
+
sys_conf = sv.instance_variable_get(:@system_config)
|
166
169
|
|
167
|
-
|
170
|
+
server = DummyServer.new
|
171
|
+
server.rpc_endpoint = sys_conf.rpc_endpoint
|
172
|
+
server.enable_get_dump = sys_conf.enable_get_dump
|
168
173
|
|
169
|
-
|
170
|
-
Net::HTTP.get URI.parse('http://0.0.0.0:24447/api/plugins.flushBuffers')
|
171
|
-
info_msg = '[info]: force flushing buffered events' + "\n"
|
174
|
+
server.run_rpc_server
|
172
175
|
|
173
|
-
|
176
|
+
sv.send(:install_main_process_signal_handlers)
|
177
|
+
Net::HTTP.get URI.parse('http://0.0.0.0:24447/api/plugins.flushBuffers')
|
178
|
+
info_msg = '[info]: force flushing buffered events' + "\n"
|
174
179
|
|
175
|
-
|
176
|
-
# This test will be passed in such environment.
|
177
|
-
pend unless $log.out.logs.first
|
180
|
+
server.stop_rpc_server
|
178
181
|
|
179
|
-
|
180
|
-
|
182
|
+
# In TravisCI with OSX(Xcode), it seems that can't use rpc server.
|
183
|
+
# This test will be passed in such environment.
|
184
|
+
pend unless $log.out.logs.first
|
181
185
|
|
186
|
+
assert{ $log.out.logs.first.end_with?(info_msg) }
|
187
|
+
ensure
|
182
188
|
$log.out.reset
|
183
189
|
end
|
184
190
|
|
@@ -197,6 +203,7 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
197
203
|
write_config tmp_dir, conf_info_str
|
198
204
|
|
199
205
|
params = {}
|
206
|
+
params['workers'] = 1
|
200
207
|
params['use_v1_config'] = true
|
201
208
|
params['log_path'] = 'test/tmp/supervisor/log'
|
202
209
|
params['suppress_repeated_stacktrace'] = true
|
@@ -243,7 +250,7 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
243
250
|
|
244
251
|
# fifth call after changed conf file(don't reuse config)
|
245
252
|
se_config = load_config_proc.call
|
246
|
-
assert_equal Fluent::Log::
|
253
|
+
assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
|
247
254
|
end
|
248
255
|
|
249
256
|
def test_load_config_for_daemonize
|
@@ -261,12 +268,13 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
261
268
|
write_config tmp_dir, conf_info_str
|
262
269
|
|
263
270
|
params = {}
|
271
|
+
params['workers'] = 1
|
264
272
|
params['use_v1_config'] = true
|
265
273
|
params['log_path'] = 'test/tmp/supervisor/log'
|
266
274
|
params['suppress_repeated_stacktrace'] = true
|
267
275
|
params['log_level'] = Fluent::Log::LEVEL_INFO
|
268
276
|
params['daemonize'] = './fluentd.pid'
|
269
|
-
load_config_proc =
|
277
|
+
load_config_proc = Proc.new { Fluent::Supervisor.load_config(tmp_dir, params) }
|
270
278
|
|
271
279
|
# first call
|
272
280
|
se_config = load_config_proc.call
|
@@ -308,14 +316,14 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
308
316
|
|
309
317
|
# fifth call after changed conf file(don't reuse config)
|
310
318
|
se_config = load_config_proc.call
|
311
|
-
assert_equal Fluent::Log::
|
319
|
+
assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
|
312
320
|
end
|
313
321
|
|
314
322
|
def test_logger
|
315
323
|
opts = Fluent::Supervisor.default_options
|
316
324
|
sv = Fluent::Supervisor.new(opts)
|
317
325
|
log = sv.instance_variable_get(:@log)
|
318
|
-
log.init
|
326
|
+
log.init(:standalone, 0)
|
319
327
|
logger = $log.instance_variable_get(:@logger)
|
320
328
|
|
321
329
|
assert_equal Fluent::Log::LEVEL_INFO, $log.level
|
@@ -340,7 +348,7 @@ class SupervisorTest < ::Test::Unit::TestCase
|
|
340
348
|
)
|
341
349
|
sv = Fluent::Supervisor.new(opts)
|
342
350
|
log = sv.instance_variable_get(:@log)
|
343
|
-
log.init
|
351
|
+
log.init(:standalone, 0)
|
344
352
|
|
345
353
|
assert_equal Fluent::LogDeviceIO, $log.out.class
|
346
354
|
assert_equal rotate_age, $log.out.instance_variable_get(:@shift_age)
|
data/test/test_test_drivers.rb
CHANGED
@@ -67,7 +67,7 @@ class TestDriverTest < ::Test::Unit::TestCase
|
|
67
67
|
assert_nothing_raised do
|
68
68
|
before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
69
69
|
d.end_if{ false }
|
70
|
-
d.run(timeout:
|
70
|
+
d.run(timeout: 5) do
|
71
71
|
sleep 0.1 until d.stop?
|
72
72
|
end
|
73
73
|
after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
@@ -89,7 +89,7 @@ class TestDriverTest < ::Test::Unit::TestCase
|
|
89
89
|
end
|
90
90
|
end
|
91
91
|
end
|
92
|
-
assert_raise "yaaaaaaaaaay!" do
|
92
|
+
assert_raise RuntimeError.new("yaaaaaaaaaay!") do
|
93
93
|
d.end_if{ false }
|
94
94
|
d.run(timeout: 3) do
|
95
95
|
sleep 0.1 until d.stop?
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.14.
|
4
|
+
version: 0.14.12
|
5
5
|
platform: x86-mingw32
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -369,17 +369,21 @@ files:
|
|
369
369
|
- code-of-conduct.md
|
370
370
|
- example/copy_roundrobin.conf
|
371
371
|
- example/filter_stdout.conf
|
372
|
+
- example/in_dummy_blocks.conf
|
372
373
|
- example/in_dummy_with_compression.conf
|
373
374
|
- example/in_forward.conf
|
374
375
|
- example/in_forward_client.conf
|
375
376
|
- example/in_forward_shared_key.conf
|
377
|
+
- example/in_forward_tls.conf
|
376
378
|
- example/in_forward_users.conf
|
379
|
+
- example/in_forward_workers.conf
|
377
380
|
- example/in_http.conf
|
378
381
|
- example/in_out_forward.conf
|
379
382
|
- example/in_syslog.conf
|
380
383
|
- example/in_tail.conf
|
381
384
|
- example/in_tcp.conf
|
382
385
|
- example/in_udp.conf
|
386
|
+
- example/logevents.conf
|
383
387
|
- example/multi_filters.conf
|
384
388
|
- example/out_copy.conf
|
385
389
|
- example/out_exec_filter.conf
|
@@ -387,15 +391,19 @@ files:
|
|
387
391
|
- example/out_forward.conf
|
388
392
|
- example/out_forward_buf_file.conf
|
389
393
|
- example/out_forward_client.conf
|
394
|
+
- example/out_forward_heartbeat_none.conf
|
390
395
|
- example/out_forward_shared_key.conf
|
396
|
+
- example/out_forward_tls.conf
|
391
397
|
- example/out_forward_users.conf
|
392
398
|
- example/out_null.conf
|
393
399
|
- example/secondary_file.conf
|
400
|
+
- example/suppress_config_dump.conf
|
394
401
|
- example/v0_12_filter.conf
|
395
402
|
- example/v1_literal_example.conf
|
396
403
|
- fluent.conf
|
397
404
|
- fluentd.gemspec
|
398
405
|
- lib/fluent/agent.rb
|
406
|
+
- lib/fluent/clock.rb
|
399
407
|
- lib/fluent/command/binlog_reader.rb
|
400
408
|
- lib/fluent/command/bundler_injection.rb
|
401
409
|
- lib/fluent/command/cat.rb
|
@@ -527,6 +535,7 @@ files:
|
|
527
535
|
- lib/fluent/plugin/storage_local.rb
|
528
536
|
- lib/fluent/plugin/string_util.rb
|
529
537
|
- lib/fluent/plugin_helper.rb
|
538
|
+
- lib/fluent/plugin_helper/cert_option.rb
|
530
539
|
- lib/fluent/plugin_helper/child_process.rb
|
531
540
|
- lib/fluent/plugin_helper/compat_parameters.rb
|
532
541
|
- lib/fluent/plugin_helper/event_emitter.rb
|
@@ -543,7 +552,6 @@ files:
|
|
543
552
|
- lib/fluent/plugin_helper/thread.rb
|
544
553
|
- lib/fluent/plugin_helper/timer.rb
|
545
554
|
- lib/fluent/plugin_id.rb
|
546
|
-
- lib/fluent/process.rb
|
547
555
|
- lib/fluent/registry.rb
|
548
556
|
- lib/fluent/root_agent.rb
|
549
557
|
- lib/fluent/rpc.rb
|
@@ -612,6 +620,7 @@ files:
|
|
612
620
|
- test/plugin/test_filter_record_transformer.rb
|
613
621
|
- test/plugin/test_filter_stdout.rb
|
614
622
|
- test/plugin/test_formatter_csv.rb
|
623
|
+
- test/plugin/test_formatter_hash.rb
|
615
624
|
- test/plugin/test_formatter_json.rb
|
616
625
|
- test/plugin/test_formatter_ltsv.rb
|
617
626
|
- test/plugin/test_formatter_msgpack.rb
|
@@ -687,6 +696,7 @@ files:
|
|
687
696
|
- test/scripts/fluent/plugin/out_test.rb
|
688
697
|
- test/scripts/fluent/plugin/out_test2.rb
|
689
698
|
- test/scripts/fluent/plugin/parser_known.rb
|
699
|
+
- test/test_clock.rb
|
690
700
|
- test/test_config.rb
|
691
701
|
- test/test_configdsl.rb
|
692
702
|
- test/test_event.rb
|
@@ -699,10 +709,10 @@ files:
|
|
699
709
|
- test/test_match.rb
|
700
710
|
- test/test_mixin.rb
|
701
711
|
- test/test_output.rb
|
712
|
+
- test/test_plugin.rb
|
702
713
|
- test/test_plugin_classes.rb
|
703
714
|
- test/test_plugin_helper.rb
|
704
715
|
- test/test_plugin_id.rb
|
705
|
-
- test/test_process.rb
|
706
716
|
- test/test_root_agent.rb
|
707
717
|
- test/test_supervisor.rb
|
708
718
|
- test/test_test_drivers.rb
|
@@ -729,7 +739,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
729
739
|
version: '0'
|
730
740
|
requirements: []
|
731
741
|
rubyforge_project:
|
732
|
-
rubygems_version: 2.
|
742
|
+
rubygems_version: 2.6.8
|
733
743
|
signing_key:
|
734
744
|
specification_version: 4
|
735
745
|
summary: Fluentd event collector
|
@@ -771,6 +781,7 @@ test_files:
|
|
771
781
|
- test/plugin/test_filter_record_transformer.rb
|
772
782
|
- test/plugin/test_filter_stdout.rb
|
773
783
|
- test/plugin/test_formatter_csv.rb
|
784
|
+
- test/plugin/test_formatter_hash.rb
|
774
785
|
- test/plugin/test_formatter_json.rb
|
775
786
|
- test/plugin/test_formatter_ltsv.rb
|
776
787
|
- test/plugin/test_formatter_msgpack.rb
|
@@ -846,6 +857,7 @@ test_files:
|
|
846
857
|
- test/scripts/fluent/plugin/out_test.rb
|
847
858
|
- test/scripts/fluent/plugin/out_test2.rb
|
848
859
|
- test/scripts/fluent/plugin/parser_known.rb
|
860
|
+
- test/test_clock.rb
|
849
861
|
- test/test_config.rb
|
850
862
|
- test/test_configdsl.rb
|
851
863
|
- test/test_event.rb
|
@@ -858,14 +870,13 @@ test_files:
|
|
858
870
|
- test/test_match.rb
|
859
871
|
- test/test_mixin.rb
|
860
872
|
- test/test_output.rb
|
873
|
+
- test/test_plugin.rb
|
861
874
|
- test/test_plugin_classes.rb
|
862
875
|
- test/test_plugin_helper.rb
|
863
876
|
- test/test_plugin_id.rb
|
864
|
-
- test/test_process.rb
|
865
877
|
- test/test_root_agent.rb
|
866
878
|
- test/test_supervisor.rb
|
867
879
|
- test/test_test_drivers.rb
|
868
880
|
- test/test_time_formatter.rb
|
869
881
|
- test/test_time_parser.rb
|
870
882
|
- test/test_unique_id.rb
|
871
|
-
has_rdoc: false
|
data/lib/fluent/process.rb
DELETED
@@ -1,504 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Fluentd
|
3
|
-
#
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
-
# you may not use this file except in compliance with the License.
|
6
|
-
# You may obtain a copy of the License at
|
7
|
-
#
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
-
#
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
-
# See the License for the specific language governing permissions and
|
14
|
-
# limitations under the License.
|
15
|
-
#
|
16
|
-
|
17
|
-
# This feature will be deprecated after introducing
|
18
|
-
# symmetric multi processing in core.
|
19
|
-
|
20
|
-
require 'thread'
|
21
|
-
|
22
|
-
require 'fluent/config'
|
23
|
-
## This comment out (to remove circular reference) is a bit risky,
|
24
|
-
## but in all known cases this file is required after 'fluent/engine'.
|
25
|
-
# require 'fluent/engine'
|
26
|
-
require 'fluent/event'
|
27
|
-
|
28
|
-
module Fluent
|
29
|
-
class DetachProcessManager
|
30
|
-
require 'singleton'
|
31
|
-
include Singleton
|
32
|
-
|
33
|
-
class Broker
|
34
|
-
def initialize
|
35
|
-
end
|
36
|
-
|
37
|
-
def engine
|
38
|
-
Engine
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def initialize
|
43
|
-
require 'drb'
|
44
|
-
DRb.start_service(create_drb_uri, Broker.new)
|
45
|
-
@parent_uri = DRb.uri
|
46
|
-
end
|
47
|
-
|
48
|
-
def fork(delegate_object)
|
49
|
-
ipr, ipw = IO.pipe # child Engine.emit_stream -> parent Engine.emit_stream
|
50
|
-
opr, opw = IO.pipe # parent target.emit_events -> child target.emit_events
|
51
|
-
|
52
|
-
pid = Process.fork
|
53
|
-
if pid
|
54
|
-
# parent process
|
55
|
-
ipw.close
|
56
|
-
opr.close
|
57
|
-
forward_thread = process_parent(ipr, opw, pid, delegate_object)
|
58
|
-
return pid, forward_thread
|
59
|
-
end
|
60
|
-
|
61
|
-
# child process
|
62
|
-
ipr.close
|
63
|
-
opw.close
|
64
|
-
forward_thread = process_child(ipw, opr, delegate_object)
|
65
|
-
return nil, forward_thread
|
66
|
-
end
|
67
|
-
|
68
|
-
private
|
69
|
-
def read_header(ipr)
|
70
|
-
sz = ipr.read(4).unpack('N')[0]
|
71
|
-
ipr.read(sz)
|
72
|
-
end
|
73
|
-
|
74
|
-
def send_header(ipw, data)
|
75
|
-
ipw.write [data.bytesize].pack('N')
|
76
|
-
ipw.write data
|
77
|
-
ipw.flush
|
78
|
-
end
|
79
|
-
|
80
|
-
def create_drb_uri
|
81
|
-
"drbunix:" # TODO
|
82
|
-
end
|
83
|
-
|
84
|
-
private
|
85
|
-
def process_child(ipw, opr, delegate_object)
|
86
|
-
DRb.start_service(create_drb_uri, delegate_object)
|
87
|
-
child_uri = DRb.uri
|
88
|
-
|
89
|
-
send_header(ipw, child_uri)
|
90
|
-
|
91
|
-
# override target.emit_stream to write event stream to the pipe
|
92
|
-
fwd = new_forwarder(ipw, 0.5) # TODO interval
|
93
|
-
Engine.define_singleton_method(:emit_stream) do |tag,es|
|
94
|
-
fwd.emit(tag, es)
|
95
|
-
end
|
96
|
-
|
97
|
-
# read event stream from the pipe and forward to target.emit
|
98
|
-
forward_thread = Thread.new(opr, delegate_object, &method(:output_forward_main))
|
99
|
-
|
100
|
-
# override global methods to call parent process
|
101
|
-
override_shared_methods(@parent_uri)
|
102
|
-
|
103
|
-
return forward_thread
|
104
|
-
end
|
105
|
-
|
106
|
-
def override_shared_methods(parent_uri)
|
107
|
-
broker = DRbObject.new_with_uri(parent_uri)
|
108
|
-
shared_methods.each {|(broker_accessor,target,name)|
|
109
|
-
remote = broker.send(broker_accessor)
|
110
|
-
target.define_singleton_method(name) do |*args,&block|
|
111
|
-
remote.send(name, *args, &block)
|
112
|
-
end
|
113
|
-
}
|
114
|
-
end
|
115
|
-
|
116
|
-
def shared_methods
|
117
|
-
[
|
118
|
-
#[:engine, Engine, :flush!],
|
119
|
-
#[:engine, Engine, :stop],
|
120
|
-
]
|
121
|
-
end
|
122
|
-
|
123
|
-
def process_parent(ipr, opw, pid, delegate_object)
|
124
|
-
# child_uri = read_header(ipr)
|
125
|
-
|
126
|
-
# read event stream from the pipe and forward to Engine.emit_stream
|
127
|
-
forward_thread = Thread.new(ipr, pid, &method(:input_forward_main))
|
128
|
-
|
129
|
-
# note: don't override methods in parent process
|
130
|
-
# because another process may fork after overriding
|
131
|
-
#override_delegate_methods(delegate_object, child_uri)
|
132
|
-
|
133
|
-
# return forwarder for DetachProcessMixin to
|
134
|
-
# override target.emit and write event stream to the pipe
|
135
|
-
fwd = new_forwarder(opw, 0.5) # TODO interval
|
136
|
-
# note: override emit method on DetachProcessMixin
|
137
|
-
forward_thread.define_singleton_method(:forwarder) do
|
138
|
-
fwd
|
139
|
-
end
|
140
|
-
|
141
|
-
return forward_thread
|
142
|
-
end
|
143
|
-
|
144
|
-
#def override_delegate_methods(target, child_uri)
|
145
|
-
# remote = DRbObject.new_with_uri(child_uri)
|
146
|
-
# delegate_methods(target).each {|name|
|
147
|
-
# target.define_singleton_method(name) do |*args,&block|
|
148
|
-
# remote.send(name, *args, &block)
|
149
|
-
# end
|
150
|
-
# }
|
151
|
-
#end
|
152
|
-
#
|
153
|
-
#def delegate_methods(target)
|
154
|
-
# target.methods - Object.public_instance_methods
|
155
|
-
#end
|
156
|
-
|
157
|
-
def output_forward_main(opr, target)
|
158
|
-
read_event_stream(opr) {|tag,es|
|
159
|
-
# FIXME error handling
|
160
|
-
begin
|
161
|
-
target.emit_events(tag, es)
|
162
|
-
rescue
|
163
|
-
$log.warn "failed to emit", error: $!.to_s, pid: Process.pid
|
164
|
-
$log.warn_backtrace
|
165
|
-
end
|
166
|
-
}
|
167
|
-
rescue
|
168
|
-
$log.error "error on output process forwarding thread", error: $!.to_s, pid: Process.pid
|
169
|
-
$log.error_backtrace
|
170
|
-
raise
|
171
|
-
end
|
172
|
-
|
173
|
-
def input_forward_main(ipr, pid)
|
174
|
-
read_event_stream(ipr) {|tag,es|
|
175
|
-
# FIXME error handling
|
176
|
-
begin
|
177
|
-
Engine.emit_stream(tag, es)
|
178
|
-
rescue
|
179
|
-
$log.warn "failed to emit", error: $!.to_s, pid: Process.pid
|
180
|
-
$log.warn_backtrace
|
181
|
-
end
|
182
|
-
}
|
183
|
-
rescue
|
184
|
-
$log.error "error on input process forwarding thread", error: $!.to_s, pid: Process.pid
|
185
|
-
$log.error_backtrace
|
186
|
-
raise
|
187
|
-
end
|
188
|
-
|
189
|
-
def read_event_stream(r, &block)
|
190
|
-
u = Fluent::Engine.msgpack_factory.unpacker(r)
|
191
|
-
begin
|
192
|
-
#buf = ''
|
193
|
-
#map = {}
|
194
|
-
#while true
|
195
|
-
# r.readpartial(64*1024, buf)
|
196
|
-
# u.feed_each(buf) {|tag,ms|
|
197
|
-
# if msbuf = map[tag]
|
198
|
-
# msbuf << ms
|
199
|
-
# else
|
200
|
-
# map[tag] = ms
|
201
|
-
# end
|
202
|
-
# }
|
203
|
-
# unless map.empty?
|
204
|
-
# map.each_pair {|tag,ms|
|
205
|
-
# es = MessagePackEventStream.new(ms)
|
206
|
-
# block.call(tag, es)
|
207
|
-
# }
|
208
|
-
# map.clear
|
209
|
-
# end
|
210
|
-
#end
|
211
|
-
u.each {|tag,ms|
|
212
|
-
es = MessagePackEventStream.new(ms)
|
213
|
-
block.call(tag, es)
|
214
|
-
}
|
215
|
-
rescue EOFError
|
216
|
-
ensure
|
217
|
-
r.close
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
def new_forwarder(w, interval)
|
222
|
-
if interval < 0.2 # TODO interval
|
223
|
-
Forwarder.new(w)
|
224
|
-
else
|
225
|
-
DelayedForwarder.new(w, interval)
|
226
|
-
end
|
227
|
-
end
|
228
|
-
|
229
|
-
class Forwarder
|
230
|
-
def initialize(w)
|
231
|
-
@w = w
|
232
|
-
end
|
233
|
-
|
234
|
-
def emit(tag, es)
|
235
|
-
ms = es.to_msgpack_stream
|
236
|
-
#[tag, ms].to_msgpack(@w) # not thread safe
|
237
|
-
@w.write [tag, ms].to_msgpack
|
238
|
-
end
|
239
|
-
end
|
240
|
-
|
241
|
-
class DelayedForwarder
|
242
|
-
def initialize(w, interval)
|
243
|
-
@w = w
|
244
|
-
@interval = interval
|
245
|
-
@buffer = {}
|
246
|
-
@mutex = Mutex.new
|
247
|
-
Thread.new(&method(:run))
|
248
|
-
end
|
249
|
-
|
250
|
-
def emit(tag, es)
|
251
|
-
stream = es.to_msgpack_stream
|
252
|
-
@mutex.synchronize do
|
253
|
-
if @buffer[tag]
|
254
|
-
@buffer[tag] << stream
|
255
|
-
else
|
256
|
-
@buffer[tag] = stream
|
257
|
-
end
|
258
|
-
end
|
259
|
-
end
|
260
|
-
|
261
|
-
def run
|
262
|
-
while true
|
263
|
-
sleep @interval
|
264
|
-
|
265
|
-
pairs = []
|
266
|
-
@mutex.synchronize do
|
267
|
-
@buffer.keys.each do |tag|
|
268
|
-
if ms = @buffer.delete(tag)
|
269
|
-
pairs << [tag, ms]
|
270
|
-
end
|
271
|
-
end
|
272
|
-
end
|
273
|
-
pairs.each do |pair|
|
274
|
-
pair.to_msgpack(@w)
|
275
|
-
end
|
276
|
-
end
|
277
|
-
rescue
|
278
|
-
$log.error "error on forwerder thread", error: $!.to_s
|
279
|
-
$log.error_backtrace
|
280
|
-
raise
|
281
|
-
end
|
282
|
-
end
|
283
|
-
|
284
|
-
class MultiForwarder
|
285
|
-
def initialize(forwarders)
|
286
|
-
@forwarders = forwarders
|
287
|
-
@rr = 1
|
288
|
-
end
|
289
|
-
|
290
|
-
def emit(tag, es)
|
291
|
-
forwarder = @forwarders[@rr]
|
292
|
-
@rr = (@rr + 1) % @forwarders.length
|
293
|
-
forwarder.emit(tag, es)
|
294
|
-
end
|
295
|
-
end
|
296
|
-
end
|
297
|
-
|
298
|
-
|
299
|
-
module DetachProcessImpl
|
300
|
-
def on_detach_process(i)
|
301
|
-
end
|
302
|
-
|
303
|
-
def on_exit_process(i)
|
304
|
-
end
|
305
|
-
|
306
|
-
private
|
307
|
-
|
308
|
-
def detach_process_impl(num, &block)
|
309
|
-
children = []
|
310
|
-
|
311
|
-
num.times do |i|
|
312
|
-
pid, forward_thread = DetachProcessManager.instance.fork(self)
|
313
|
-
|
314
|
-
if pid
|
315
|
-
# parent process
|
316
|
-
$log.info "detached process", class: self.class, pid: pid
|
317
|
-
children << [pid, forward_thread]
|
318
|
-
next
|
319
|
-
end
|
320
|
-
|
321
|
-
# child process
|
322
|
-
begin
|
323
|
-
on_detach_process(i)
|
324
|
-
|
325
|
-
block.call
|
326
|
-
|
327
|
-
# disable Engine.stop called by signal handler
|
328
|
-
Engine.define_singleton_method(:stop) do
|
329
|
-
# do nothing
|
330
|
-
end
|
331
|
-
|
332
|
-
# override signal handlers called by parent process
|
333
|
-
fin = FinishWait.new
|
334
|
-
trap :INT do
|
335
|
-
fin.stop
|
336
|
-
end
|
337
|
-
trap :TERM do
|
338
|
-
fin.stop
|
339
|
-
end
|
340
|
-
#forward_thread.join # TODO this thread won't stop because parent doesn't close pipe
|
341
|
-
fin.wait
|
342
|
-
|
343
|
-
on_exit_process(i)
|
344
|
-
exit! 0
|
345
|
-
ensure
|
346
|
-
$log.error "unknown error while shutting down this child process", error: $!.to_s, pid: Process.pid
|
347
|
-
$log.error_backtrace
|
348
|
-
end
|
349
|
-
|
350
|
-
exit! 1
|
351
|
-
end
|
352
|
-
|
353
|
-
# parent process
|
354
|
-
# override shutdown method to kill child processes
|
355
|
-
define_singleton_method(:shutdown) do
|
356
|
-
children.each {|pair|
|
357
|
-
begin
|
358
|
-
pid = pair[0]
|
359
|
-
forward_thread = pair[1]
|
360
|
-
if pid
|
361
|
-
Process.kill(:TERM, pid)
|
362
|
-
forward_thread.join # wait until child closes pipe
|
363
|
-
Process.waitpid(pid)
|
364
|
-
pair[0] = nil
|
365
|
-
end
|
366
|
-
rescue
|
367
|
-
$log.error "unknown error while shutting down remote child process", error: $!.to_s
|
368
|
-
$log.error_backtrace
|
369
|
-
end
|
370
|
-
}
|
371
|
-
end
|
372
|
-
|
373
|
-
# override target.emit_events and write event stream to the pipe
|
374
|
-
forwarders = children.map {|pair| pair[1].forwarder }
|
375
|
-
if forwarders.length > 1
|
376
|
-
# use roundrobin
|
377
|
-
fwd = DetachProcessManager::MultiForwarder.new(forwarders)
|
378
|
-
else
|
379
|
-
fwd = forwarders[0]
|
380
|
-
end
|
381
|
-
define_singleton_method(:emit_events) do |tag,es|
|
382
|
-
fwd.emit_events(tag, es)
|
383
|
-
end
|
384
|
-
end
|
385
|
-
|
386
|
-
class FinishWait
|
387
|
-
def initialize
|
388
|
-
@finished = false
|
389
|
-
@mutex = Mutex.new
|
390
|
-
@cond = ConditionVariable.new
|
391
|
-
end
|
392
|
-
|
393
|
-
def wait
|
394
|
-
@mutex.synchronize do
|
395
|
-
until @finished
|
396
|
-
@cond.wait(@mutex, 1.0)
|
397
|
-
end
|
398
|
-
end
|
399
|
-
end
|
400
|
-
|
401
|
-
def stop
|
402
|
-
return if @finished
|
403
|
-
@finished = true
|
404
|
-
# Creating new thread due to mutex can't lock in main thread during trap context
|
405
|
-
Thread.new {
|
406
|
-
@mutex.synchronize do
|
407
|
-
@cond.broadcast
|
408
|
-
end
|
409
|
-
}.run
|
410
|
-
end
|
411
|
-
|
412
|
-
def finished?
|
413
|
-
@finished
|
414
|
-
end
|
415
|
-
end
|
416
|
-
end
|
417
|
-
|
418
|
-
|
419
|
-
module DetachProcessMixin
|
420
|
-
include DetachProcessImpl
|
421
|
-
|
422
|
-
def configure(conf)
|
423
|
-
super
|
424
|
-
|
425
|
-
@detach_process = nil
|
426
|
-
|
427
|
-
if detach_process = conf['detach_process']
|
428
|
-
b3v = Config.bool_value(detach_process)
|
429
|
-
case b3v
|
430
|
-
when nil
|
431
|
-
num = detach_process.to_i
|
432
|
-
if num > 1
|
433
|
-
$log.warn "'detach_process' parameter supports only 1 process on this plugin: #{conf}"
|
434
|
-
elsif num > 0
|
435
|
-
@detach_process = true
|
436
|
-
elsif detach_process =~ /0+/
|
437
|
-
@detach_process = false
|
438
|
-
else
|
439
|
-
@detach_process = true
|
440
|
-
end
|
441
|
-
when true
|
442
|
-
@detach_process = true
|
443
|
-
when false
|
444
|
-
@detach_process = false
|
445
|
-
end
|
446
|
-
end
|
447
|
-
end
|
448
|
-
|
449
|
-
def detach_process(&block)
|
450
|
-
if @detach_process
|
451
|
-
detach_process_impl(1, &block)
|
452
|
-
else
|
453
|
-
block.call
|
454
|
-
end
|
455
|
-
end
|
456
|
-
end
|
457
|
-
|
458
|
-
|
459
|
-
module DetachMultiProcessMixin
|
460
|
-
include DetachProcessImpl
|
461
|
-
|
462
|
-
def initialize
|
463
|
-
@detach_process_num = 2
|
464
|
-
super
|
465
|
-
end
|
466
|
-
|
467
|
-
def configure(conf)
|
468
|
-
super
|
469
|
-
|
470
|
-
@detach_process = nil
|
471
|
-
|
472
|
-
if detach_process = conf['detach_process']
|
473
|
-
b3v = Config.bool_value(detach_process)
|
474
|
-
case b3v
|
475
|
-
when nil
|
476
|
-
num = detach_process.to_i
|
477
|
-
if num > 0
|
478
|
-
@detach_process = true
|
479
|
-
@detach_process_num = num
|
480
|
-
elsif detach_process =~ /0+/
|
481
|
-
@detach_process = false
|
482
|
-
else
|
483
|
-
@detach_process = true
|
484
|
-
end
|
485
|
-
when true
|
486
|
-
@detach_process = true
|
487
|
-
when false
|
488
|
-
@detach_process = false
|
489
|
-
end
|
490
|
-
end
|
491
|
-
end
|
492
|
-
|
493
|
-
private
|
494
|
-
|
495
|
-
def detach_multi_process(&block)
|
496
|
-
if @detach_process
|
497
|
-
detach_process_impl(@detach_process_num, &block)
|
498
|
-
else
|
499
|
-
block.call
|
500
|
-
end
|
501
|
-
end
|
502
|
-
end
|
503
|
-
end
|
504
|
-
|