fluentd 0.12.40 → 0.14.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +2 -0
- data/.travis.yml +33 -21
- data/CONTRIBUTING.md +1 -0
- data/ChangeLog +810 -237
- data/README.md +0 -25
- data/Rakefile +2 -1
- data/Vagrantfile +17 -0
- data/appveyor.yml +35 -0
- data/example/filter_stdout.conf +5 -5
- data/example/in_forward.conf +2 -2
- data/example/in_http.conf +2 -2
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +2 -2
- data/example/in_tail.conf +2 -2
- data/example/in_tcp.conf +2 -2
- data/example/in_udp.conf +2 -2
- data/example/out_copy.conf +4 -4
- data/example/out_file.conf +2 -2
- data/example/out_forward.conf +2 -2
- data/example/out_forward_buf_file.conf +23 -0
- data/example/v0_12_filter.conf +8 -8
- data/fluent.conf +29 -0
- data/fluentd.gemspec +18 -11
- data/lib/fluent/agent.rb +60 -58
- data/lib/fluent/command/cat.rb +1 -1
- data/lib/fluent/command/debug.rb +7 -5
- data/lib/fluent/command/fluentd.rb +97 -2
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/filter.rb +50 -0
- data/lib/fluent/compat/formatter.rb +109 -0
- data/lib/fluent/compat/input.rb +50 -0
- data/lib/fluent/compat/output.rb +617 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +163 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/config.rb +23 -20
- data/lib/fluent/config/configure_proxy.rb +119 -70
- data/lib/fluent/config/dsl.rb +5 -18
- data/lib/fluent/config/element.rb +72 -8
- data/lib/fluent/config/error.rb +0 -3
- data/lib/fluent/config/literal_parser.rb +0 -2
- data/lib/fluent/config/parser.rb +4 -4
- data/lib/fluent/config/section.rb +39 -28
- data/lib/fluent/config/types.rb +2 -13
- data/lib/fluent/config/v1_parser.rb +1 -3
- data/lib/fluent/configurable.rb +48 -16
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +26 -52
- data/lib/fluent/env.rb +6 -4
- data/lib/fluent/event.rb +58 -11
- data/lib/fluent/event_router.rb +5 -5
- data/lib/fluent/filter.rb +2 -50
- data/lib/fluent/formatter.rb +4 -293
- data/lib/fluent/input.rb +2 -32
- data/lib/fluent/label.rb +2 -2
- data/lib/fluent/load.rb +3 -2
- data/lib/fluent/log.rb +107 -38
- data/lib/fluent/match.rb +0 -36
- data/lib/fluent/mixin.rb +117 -7
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +7 -612
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +4 -800
- data/lib/fluent/plugin.rb +100 -121
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +121 -0
- data/lib/fluent/plugin/buf_file.rb +101 -182
- data/lib/fluent/plugin/buf_memory.rb +9 -92
- data/lib/fluent/plugin/buffer.rb +473 -0
- data/lib/fluent/plugin/buffer/chunk.rb +135 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +339 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +100 -0
- data/lib/fluent/plugin/exec_util.rb +80 -75
- data/lib/fluent/plugin/file_util.rb +33 -28
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +51 -0
- data/lib/fluent/plugin/filter_grep.rb +13 -40
- data/lib/fluent/plugin/filter_record_transformer.rb +22 -18
- data/lib/fluent/plugin/formatter.rb +93 -0
- data/lib/fluent/plugin/formatter_csv.rb +48 -0
- data/lib/fluent/plugin/formatter_hash.rb +32 -0
- data/lib/fluent/plugin/formatter_json.rb +47 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +32 -0
- data/lib/fluent/plugin/formatter_out_file.rb +45 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +39 -0
- data/lib/fluent/plugin/in_debug_agent.rb +4 -0
- data/lib/fluent/plugin/in_dummy.rb +22 -18
- data/lib/fluent/plugin/in_exec.rb +18 -8
- data/lib/fluent/plugin/in_forward.rb +36 -79
- data/lib/fluent/plugin/in_gc_stat.rb +4 -0
- data/lib/fluent/plugin/in_http.rb +21 -18
- data/lib/fluent/plugin/in_monitor_agent.rb +15 -48
- data/lib/fluent/plugin/in_object_space.rb +6 -1
- data/lib/fluent/plugin/in_stream.rb +7 -3
- data/lib/fluent/plugin/in_syslog.rb +46 -95
- data/lib/fluent/plugin/in_tail.rb +51 -595
- data/lib/fluent/plugin/in_tcp.rb +8 -1
- data/lib/fluent/plugin/in_udp.rb +8 -14
- data/lib/fluent/plugin/input.rb +33 -0
- data/lib/fluent/plugin/multi_output.rb +95 -0
- data/lib/fluent/plugin/out_buffered_null.rb +59 -0
- data/lib/fluent/plugin/out_copy.rb +11 -7
- data/lib/fluent/plugin/out_exec.rb +15 -11
- data/lib/fluent/plugin/out_exec_filter.rb +18 -10
- data/lib/fluent/plugin/out_file.rb +34 -5
- data/lib/fluent/plugin/out_forward.rb +19 -9
- data/lib/fluent/plugin/out_null.rb +0 -14
- data/lib/fluent/plugin/out_roundrobin.rb +11 -7
- data/lib/fluent/plugin/out_stdout.rb +5 -7
- data/lib/fluent/plugin/out_stream.rb +3 -1
- data/lib/fluent/plugin/output.rb +979 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +244 -0
- data/lib/fluent/plugin/parser_apache.rb +24 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +21 -0
- data/lib/fluent/plugin/parser_csv.rb +31 -0
- data/lib/fluent/plugin/parser_json.rb +79 -0
- data/lib/fluent/plugin/parser_ltsv.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +102 -0
- data/lib/fluent/plugin/parser_nginx.rb +24 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_syslog.rb +82 -0
- data/lib/fluent/plugin/parser_tsv.rb +37 -0
- data/lib/fluent/plugin/socket_util.rb +120 -114
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +116 -0
- data/lib/fluent/plugin/string_util.rb +16 -13
- data/lib/fluent/plugin_helper.rb +39 -0
- data/lib/fluent/plugin_helper/child_process.rb +298 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +99 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
- data/lib/fluent/plugin_helper/event_loop.rb +118 -0
- data/lib/fluent/plugin_helper/retry_state.rb +177 -0
- data/lib/fluent/plugin_helper/storage.rb +308 -0
- data/lib/fluent/plugin_helper/thread.rb +147 -0
- data/lib/fluent/plugin_helper/timer.rb +85 -0
- data/lib/fluent/plugin_id.rb +63 -0
- data/lib/fluent/process.rb +21 -30
- data/lib/fluent/registry.rb +21 -9
- data/lib/fluent/root_agent.rb +115 -40
- data/lib/fluent/supervisor.rb +330 -320
- data/lib/fluent/system_config.rb +42 -18
- data/lib/fluent/test.rb +6 -1
- data/lib/fluent/test/base.rb +23 -3
- data/lib/fluent/test/driver/base.rb +247 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +35 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/output.rb +78 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +0 -1
- data/lib/fluent/test/formatter_test.rb +2 -1
- data/lib/fluent/test/input_test.rb +23 -17
- data/lib/fluent/test/output_test.rb +28 -39
- data/lib/fluent/test/parser_test.rb +1 -1
- data/lib/fluent/time.rb +104 -1
- data/lib/fluent/{status.rb → unique_id.rb} +15 -24
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +72 -0
- data/test/compat/test_calls_super.rb +164 -0
- data/test/config/test_config_parser.rb +83 -0
- data/test/config/test_configurable.rb +547 -274
- data/test/config/test_configure_proxy.rb +146 -29
- data/test/config/test_dsl.rb +3 -181
- data/test/config/test_element.rb +274 -0
- data/test/config/test_literal_parser.rb +1 -1
- data/test/config/test_section.rb +79 -7
- data/test/config/test_system_config.rb +21 -0
- data/test/config/test_types.rb +3 -26
- data/test/helper.rb +78 -8
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +75 -0
- data/test/plugin/test_buf_file.rb +420 -521
- data/test/plugin/test_buf_memory.rb +32 -194
- data/test/plugin/test_buffer.rb +981 -0
- data/test/plugin/test_buffer_chunk.rb +110 -0
- data/test/plugin/test_buffer_file_chunk.rb +770 -0
- data/test/plugin/test_buffer_memory_chunk.rb +265 -0
- data/test/plugin/test_filter.rb +255 -0
- data/test/plugin/test_filter_grep.rb +2 -73
- data/test/plugin/test_filter_record_transformer.rb +24 -68
- data/test/plugin/test_filter_stdout.rb +6 -6
- data/test/plugin/test_in_debug_agent.rb +2 -0
- data/test/plugin/test_in_dummy.rb +11 -17
- data/test/plugin/test_in_exec.rb +6 -25
- data/test/plugin/test_in_forward.rb +112 -151
- data/test/plugin/test_in_gc_stat.rb +2 -0
- data/test/plugin/test_in_http.rb +106 -157
- data/test/plugin/test_in_object_space.rb +21 -5
- data/test/plugin/test_in_stream.rb +14 -13
- data/test/plugin/test_in_syslog.rb +30 -275
- data/test/plugin/test_in_tail.rb +95 -234
- data/test/plugin/test_in_tcp.rb +14 -0
- data/test/plugin/test_in_udp.rb +21 -13
- data/test/plugin/test_input.rb +122 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_buffered_null.rb +79 -0
- data/test/plugin/test_out_copy.rb +15 -2
- data/test/plugin/test_out_exec.rb +75 -25
- data/test/plugin/test_out_exec_filter.rb +74 -8
- data/test/plugin/test_out_file.rb +61 -7
- data/test/plugin/test_out_forward.rb +92 -15
- data/test/plugin/test_out_roundrobin.rb +1 -0
- data/test/plugin/test_out_stdout.rb +22 -13
- data/test/plugin/test_out_stream.rb +18 -0
- data/test/plugin/test_output.rb +515 -0
- data/test/plugin/test_output_as_buffered.rb +1540 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +247 -0
- data/test/plugin/test_output_as_buffered_retries.rb +808 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +776 -0
- data/test/plugin/test_output_as_standard.rb +362 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +8 -0
- data/test/plugin_helper/test_child_process.rb +599 -0
- data/test/plugin_helper/test_compat_parameters.rb +175 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_retry_state.rb +399 -0
- data/test/plugin_helper/test_storage.rb +411 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +100 -0
- data/test/scripts/exec_script.rb +0 -6
- data/test/scripts/fluent/plugin/out_test.rb +3 -0
- data/test/test_config.rb +13 -4
- data/test/test_event.rb +24 -13
- data/test/test_event_router.rb +8 -7
- data/test/test_event_time.rb +187 -0
- data/test/test_formatter.rb +13 -51
- data/test/test_input.rb +1 -1
- data/test/test_log.rb +239 -16
- data/test/test_mixin.rb +1 -1
- data/test/test_output.rb +53 -66
- data/test/test_parser.rb +105 -323
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_root_agent.rb +4 -52
- data/test/test_supervisor.rb +272 -0
- data/test/test_unique_id.rb +47 -0
- metadata +180 -54
- data/lib/fluent/buffer.rb +0 -365
- data/lib/fluent/plugin/filter_parser.rb +0 -107
- data/lib/fluent/plugin/in_status.rb +0 -76
- data/lib/fluent/test/helpers.rb +0 -86
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/test_filter_parser.rb +0 -744
- data/test/plugin/test_in_status.rb +0 -38
- data/test/test_buffer.rb +0 -624
@@ -0,0 +1,99 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/config/element'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module PluginHelper
|
21
|
+
module CompatParameters
|
22
|
+
# This plugin helper is to bring old-fashioned buffer/other
|
23
|
+
# configuration parameters to v0.14 plugin API configurations.
|
24
|
+
# This helper is mainly to convert plugins from v0.12 API
|
25
|
+
# to v0.14 API safely, without breaking user deployment.
|
26
|
+
|
27
|
+
PARAMS_MAP = {
|
28
|
+
"buffer_type" => "@type",
|
29
|
+
"buffer_path" => "path",
|
30
|
+
"num_threads" => "flush_thread_count",
|
31
|
+
"flush_interval" => "flush_interval",
|
32
|
+
"try_flush_interval" => "flush_thread_interval",
|
33
|
+
"queued_chunk_flush_interval" => "flush_thread_burst_interval",
|
34
|
+
"disable_retry_limit" => "retry_forever",
|
35
|
+
"retry_limit" => "retry_max_times",
|
36
|
+
"max_retry_wait" => "retry_max_interval",
|
37
|
+
"buffer_chunk_limit" => "chunk_limit_size",
|
38
|
+
"buffer_queue_limit" => "queue_length_limit",
|
39
|
+
"buffer_queue_full_action" => "overflow_action",
|
40
|
+
"flush_at_shutdown" => "flush_at_shutdown",
|
41
|
+
}
|
42
|
+
|
43
|
+
TIME_SLICED_PARAMS = {
|
44
|
+
"time_slice_format" => nil,
|
45
|
+
"time_slice_wait" => "timekey_wait",
|
46
|
+
}
|
47
|
+
|
48
|
+
def compat_parameters_default_chunk_key
|
49
|
+
# '', 'time' or 'tag'
|
50
|
+
raise NotImplementedError, "return one of '', 'time' or 'tag'"
|
51
|
+
end
|
52
|
+
|
53
|
+
def configure(conf)
|
54
|
+
if conf.elements('buffer').empty? && PARAMS_MAP.keys.any?{|k| conf.has_key?(k) } || TIME_SLICED_PARAMS.keys.any?{|k| conf.has_key?(k) }
|
55
|
+
# TODO: warn obsolete parameters if these are deprecated
|
56
|
+
attr = {}
|
57
|
+
PARAMS_MAP.each do |compat, current|
|
58
|
+
next unless current
|
59
|
+
if conf.has_key?(compat)
|
60
|
+
if compat == 'buffer_queue_full_action' && conf[compat] == 'exception'
|
61
|
+
attr[current] = 'throw_exception'
|
62
|
+
else
|
63
|
+
attr[current] = conf[compat]
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
TIME_SLICED_PARAMS.each do |compat, current|
|
68
|
+
next unless current
|
69
|
+
attr[current] = conf[compat] if conf.has_key?(compat)
|
70
|
+
end
|
71
|
+
|
72
|
+
chunk_key = nil
|
73
|
+
|
74
|
+
if conf.has_key?('time_slice_format')
|
75
|
+
chunk_key = 'time'
|
76
|
+
attr['timekey'] = case conf['time_slice_format']
|
77
|
+
when /\%S/ then 1
|
78
|
+
when /\%M/ then 60
|
79
|
+
when /\%H/ then 3600
|
80
|
+
when /\%d/ then 86400
|
81
|
+
else
|
82
|
+
raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
|
83
|
+
end
|
84
|
+
else
|
85
|
+
chunk_key = compat_parameters_default_chunk_key
|
86
|
+
if chunk_key == 'time'
|
87
|
+
attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
|
92
|
+
conf.elements << e
|
93
|
+
end
|
94
|
+
|
95
|
+
super
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/time'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module PluginHelper
|
21
|
+
module EventEmitter
|
22
|
+
# stop : [-]
|
23
|
+
# shutdown : disable @router
|
24
|
+
# close : [-]
|
25
|
+
# terminate: [-]
|
26
|
+
|
27
|
+
def router
|
28
|
+
@_event_emitter_used_actually = true
|
29
|
+
@router
|
30
|
+
end
|
31
|
+
|
32
|
+
def router=(r)
|
33
|
+
@router = r
|
34
|
+
end
|
35
|
+
|
36
|
+
def has_router?
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
def event_emitter_used_actually?
|
41
|
+
@_event_emitter_used_actually
|
42
|
+
end
|
43
|
+
|
44
|
+
def event_emitter_router(label_name)
|
45
|
+
if label_name
|
46
|
+
Engine.root_agent.find_label(label_name).event_router
|
47
|
+
else
|
48
|
+
Engine.root_agent.event_router
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def initialize
|
53
|
+
super
|
54
|
+
@_event_emitter_used_actually = false
|
55
|
+
@router = nil
|
56
|
+
end
|
57
|
+
|
58
|
+
def configure(conf)
|
59
|
+
require 'fluent/engine'
|
60
|
+
super
|
61
|
+
@router = event_emitter_router(conf['@label'])
|
62
|
+
end
|
63
|
+
|
64
|
+
def after_shutdown
|
65
|
+
@router = nil
|
66
|
+
super
|
67
|
+
end
|
68
|
+
|
69
|
+
def close # unset router many times to reduce test cost
|
70
|
+
@router = nil
|
71
|
+
super
|
72
|
+
end
|
73
|
+
|
74
|
+
def terminate
|
75
|
+
@router = nil
|
76
|
+
super
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'cool.io'
|
18
|
+
require 'fluent/plugin_helper/thread'
|
19
|
+
|
20
|
+
module Fluent
|
21
|
+
module PluginHelper
|
22
|
+
module EventLoop
|
23
|
+
# Currently this plugin helper is only for other helpers, not plugins.
|
24
|
+
# there's no way to create customized watchers to attach event loops.
|
25
|
+
include Fluent::PluginHelper::Thread
|
26
|
+
|
27
|
+
# stop : [-]
|
28
|
+
# shutdown : detach all event watchers on event loop
|
29
|
+
# close : stop event loop
|
30
|
+
# terminate: initialize internal state
|
31
|
+
|
32
|
+
EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
|
33
|
+
|
34
|
+
attr_reader :_event_loop # for tests
|
35
|
+
|
36
|
+
def event_loop_attach(watcher)
|
37
|
+
@_event_loop_mutex.synchronize do
|
38
|
+
@_event_loop.attach(watcher)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def event_loop_wait_until_start
|
43
|
+
sleep(0.1) until event_loop_running?
|
44
|
+
end
|
45
|
+
|
46
|
+
def event_loop_wait_until_stop
|
47
|
+
sleep(0.1) while event_loop_running?
|
48
|
+
end
|
49
|
+
|
50
|
+
def event_loop_running?
|
51
|
+
@_event_loop_running
|
52
|
+
end
|
53
|
+
|
54
|
+
def initialize
|
55
|
+
super
|
56
|
+
@_event_loop = Coolio::Loop.new
|
57
|
+
@_event_loop_running = false
|
58
|
+
@_event_loop_mutex = Mutex.new
|
59
|
+
# plugin MAY configure loop run timeout in #configure
|
60
|
+
@_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
|
61
|
+
end
|
62
|
+
|
63
|
+
def start
|
64
|
+
super
|
65
|
+
|
66
|
+
# event loop does not run here, so mutex lock is not required
|
67
|
+
thread_create :event_loop do
|
68
|
+
default_watcher = DefaultWatcher.new
|
69
|
+
@_event_loop.attach(default_watcher)
|
70
|
+
@_event_loop_running = true
|
71
|
+
@_event_loop.run(@_event_loop_run_timeout) # this method blocks
|
72
|
+
@_event_loop_running = false
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def shutdown
|
77
|
+
@_event_loop_mutex.synchronize do
|
78
|
+
@_event_loop.watchers.each {|w| w.detach if w.attached? }
|
79
|
+
end
|
80
|
+
while @_event_loop_running
|
81
|
+
sleep 0.1
|
82
|
+
end
|
83
|
+
|
84
|
+
super
|
85
|
+
end
|
86
|
+
|
87
|
+
def close
|
88
|
+
if @_event_loop_running
|
89
|
+
begin
|
90
|
+
@_event_loop.stop # we cannot check loop is running or not
|
91
|
+
rescue RuntimeError => e
|
92
|
+
raise unless e.message == 'loop not running'
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
super
|
97
|
+
end
|
98
|
+
|
99
|
+
def terminate
|
100
|
+
@_event_loop = nil
|
101
|
+
@_event_loop_running = false
|
102
|
+
@_event_loop_mutex = nil
|
103
|
+
@_event_loop_run_timeout = nil
|
104
|
+
|
105
|
+
super
|
106
|
+
end
|
107
|
+
|
108
|
+
# watcher to block to run event loop until shutdown
|
109
|
+
class DefaultWatcher < Coolio::TimerWatcher
|
110
|
+
def initialize
|
111
|
+
super(1, true) # interval: 1, repeat: true
|
112
|
+
end
|
113
|
+
# do nothing
|
114
|
+
def on_timer; end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,177 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module PluginHelper
|
19
|
+
module RetryState
|
20
|
+
def retry_state_create(
|
21
|
+
title, retry_type, wait, timeout,
|
22
|
+
forever: false, max_steps: nil, backoff_base: 2, max_interval: nil, randomize: true, randomize_width: 0.125,
|
23
|
+
secondary: false, secondary_threshold: 0.8
|
24
|
+
)
|
25
|
+
case retry_type
|
26
|
+
when :exponential_backoff
|
27
|
+
ExponentialBackOffRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
|
28
|
+
when :periodic
|
29
|
+
PeriodicRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
|
30
|
+
else
|
31
|
+
raise "BUG: unknown retry_type specified: '#{retry_type}'"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
class RetryStateMachine
|
36
|
+
attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_steps
|
37
|
+
|
38
|
+
def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
|
39
|
+
@title = title
|
40
|
+
|
41
|
+
@start = current_time
|
42
|
+
@steps = 0
|
43
|
+
@next_time = nil # should be initialized for first retry by child class
|
44
|
+
|
45
|
+
@timeout = timeout
|
46
|
+
@timeout_at = @start + timeout
|
47
|
+
@current = :primary
|
48
|
+
|
49
|
+
if randomize_width < 0 || randomize_width > 0.5
|
50
|
+
raise "BUG: randomize_width MUST be between 0 and 0.5"
|
51
|
+
end
|
52
|
+
|
53
|
+
@randomize = randomize
|
54
|
+
@randomize_width = randomize_width
|
55
|
+
|
56
|
+
if forever && secondary
|
57
|
+
raise "BUG: forever and secondary are exclusive to each other"
|
58
|
+
end
|
59
|
+
|
60
|
+
@forever = forever
|
61
|
+
@max_steps = max_steps
|
62
|
+
|
63
|
+
@secondary = secondary
|
64
|
+
@secondary_threshold = secondary_threshold
|
65
|
+
if @secondary
|
66
|
+
raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1
|
67
|
+
@secondary_transition_at = @start + timeout * @secondary_threshold
|
68
|
+
@secondary_transition_steps = nil
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def current_time
|
73
|
+
Time.now
|
74
|
+
end
|
75
|
+
|
76
|
+
def randomize(interval)
|
77
|
+
return interval unless @randomize
|
78
|
+
|
79
|
+
interval + (interval * @randomize_width * (2 * rand - 1.0))
|
80
|
+
end
|
81
|
+
|
82
|
+
def calc_next_time
|
83
|
+
if @forever || !@secondary # primary
|
84
|
+
naive = naive_next_time(@steps)
|
85
|
+
if @forever
|
86
|
+
naive
|
87
|
+
elsif naive >= @timeout_at
|
88
|
+
@timeout_at
|
89
|
+
else
|
90
|
+
naive
|
91
|
+
end
|
92
|
+
elsif @current == :primary && @secondary
|
93
|
+
naive = naive_next_time(@steps)
|
94
|
+
if naive >= @secondary_transition_at
|
95
|
+
@secondary_transition_at
|
96
|
+
else
|
97
|
+
naive
|
98
|
+
end
|
99
|
+
elsif @current == :secondary
|
100
|
+
naive = naive_next_time(@steps - @secondary_transition_steps + 1)
|
101
|
+
if naive >= @timeout_at
|
102
|
+
@timeout_at
|
103
|
+
else
|
104
|
+
naive
|
105
|
+
end
|
106
|
+
else
|
107
|
+
raise "BUG: it's out of design"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def naive_next_time(retry_times)
|
112
|
+
raise NotImplementedError
|
113
|
+
end
|
114
|
+
|
115
|
+
def secondary?
|
116
|
+
@secondary && (@current == :secondary || current_time >= @secondary_transition_at)
|
117
|
+
end
|
118
|
+
|
119
|
+
def step
|
120
|
+
@steps += 1
|
121
|
+
if @secondary && @current != :secondary && current_time >= @secondary_transition_at
|
122
|
+
@current = :secondary
|
123
|
+
@secondary_transition_steps = @steps
|
124
|
+
end
|
125
|
+
@next_time = calc_next_time
|
126
|
+
nil
|
127
|
+
end
|
128
|
+
|
129
|
+
def limit?
|
130
|
+
if @forever
|
131
|
+
false
|
132
|
+
else
|
133
|
+
@next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
class ExponentialBackOffRetry < RetryStateMachine
|
139
|
+
def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threathold)
|
140
|
+
super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
|
141
|
+
@constant_factor = wait
|
142
|
+
@backoff_base = backoff_base
|
143
|
+
@max_interval = max_interval
|
144
|
+
|
145
|
+
@next_time = @start + @constant_factor
|
146
|
+
end
|
147
|
+
|
148
|
+
def naive_next_time(retry_next_times)
|
149
|
+
# make it infinite if calculated "interval" is too big
|
150
|
+
interval = @constant_factor.to_f * ( @backoff_base ** ( retry_next_times - 1 ) )
|
151
|
+
intr = if interval.finite?
|
152
|
+
if @max_interval && interval > @max_interval
|
153
|
+
@max_interval
|
154
|
+
else
|
155
|
+
interval
|
156
|
+
end
|
157
|
+
else
|
158
|
+
interval
|
159
|
+
end
|
160
|
+
current_time + randomize(intr)
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
class PeriodicRetry < RetryStateMachine
|
165
|
+
def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
|
166
|
+
super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
|
167
|
+
@retry_wait = wait
|
168
|
+
@next_time = @start + @retry_wait
|
169
|
+
end
|
170
|
+
|
171
|
+
def naive_next_time(retry_next_times)
|
172
|
+
current_time + randomize(@retry_wait)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|