fluentd 0.14.4-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +26 -0
- data/.travis.yml +45 -0
- data/AUTHORS +2 -0
- data/CONTRIBUTING.md +35 -0
- data/COPYING +14 -0
- data/ChangeLog +276 -0
- data/Gemfile +9 -0
- data/README.md +51 -0
- data/Rakefile +53 -0
- data/Vagrantfile +17 -0
- data/appveyor.yml +41 -0
- data/bin/fluent-debug +5 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +11 -0
- data/example/in_http.conf +14 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_buffered_null.conf +32 -0
- data/example/out_copy.conf +20 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +51 -0
- data/lib/fluent/agent.rb +194 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/cat.rb +319 -0
- data/lib/fluent/command/debug.rb +102 -0
- data/lib/fluent/command/fluentd.rb +273 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +677 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +180 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config.rb +56 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +366 -0
- data/lib/fluent/config/dsl.rb +149 -0
- data/lib/fluent/config/element.rb +218 -0
- data/lib/fluent/config/error.rb +26 -0
- data/lib/fluent/config/literal_parser.rb +251 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +212 -0
- data/lib/fluent/config/types.rb +136 -0
- data/lib/fluent/config/v1_parser.rb +190 -0
- data/lib/fluent/configurable.rb +176 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +220 -0
- data/lib/fluent/env.rb +27 -0
- data/lib/fluent/event.rb +287 -0
- data/lib/fluent/event_router.rb +259 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +38 -0
- data/lib/fluent/load.rb +36 -0
- data/lib/fluent/log.rb +445 -0
- data/lib/fluent/match.rb +141 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +26 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin.rb +161 -0
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +130 -0
- data/lib/fluent/plugin/buf_file.rb +154 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer.rb +603 -0
- data/lib/fluent/plugin/buffer/chunk.rb +160 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +75 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +45 -0
- data/lib/fluent/plugin/formatter_csv.rb +47 -0
- data/lib/fluent/plugin/formatter_hash.rb +29 -0
- data/lib/fluent/plugin/formatter_json.rb +44 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
- data/lib/fluent/plugin/formatter_out_file.rb +78 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +74 -0
- data/lib/fluent/plugin/in_debug_agent.rb +64 -0
- data/lib/fluent/plugin/in_dummy.rb +135 -0
- data/lib/fluent/plugin/in_exec.rb +149 -0
- data/lib/fluent/plugin/in_forward.rb +366 -0
- data/lib/fluent/plugin/in_gc_stat.rb +52 -0
- data/lib/fluent/plugin/in_http.rb +422 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
- data/lib/fluent/plugin/in_object_space.rb +90 -0
- data/lib/fluent/plugin/in_syslog.rb +204 -0
- data/lib/fluent/plugin/in_tail.rb +838 -0
- data/lib/fluent/plugin/in_tcp.rb +41 -0
- data/lib/fluent/plugin/in_udp.rb +37 -0
- data/lib/fluent/plugin/in_unix.rb +201 -0
- data/lib/fluent/plugin/input.rb +33 -0
- data/lib/fluent/plugin/multi_output.rb +95 -0
- data/lib/fluent/plugin/out_buffered_null.rb +59 -0
- data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
- data/lib/fluent/plugin/out_copy.rb +42 -0
- data/lib/fluent/plugin/out_exec.rb +114 -0
- data/lib/fluent/plugin/out_exec_filter.rb +393 -0
- data/lib/fluent/plugin/out_file.rb +167 -0
- data/lib/fluent/plugin/out_forward.rb +646 -0
- data/lib/fluent/plugin/out_null.rb +27 -0
- data/lib/fluent/plugin/out_relabel.rb +28 -0
- data/lib/fluent/plugin/out_roundrobin.rb +80 -0
- data/lib/fluent/plugin/out_stdout.rb +48 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1020 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +175 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +33 -0
- data/lib/fluent/plugin/parser_json.rb +79 -0
- data/lib/fluent/plugin/parser_ltsv.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +104 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +73 -0
- data/lib/fluent/plugin/parser_syslog.rb +82 -0
- data/lib/fluent/plugin/parser_tsv.rb +37 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +132 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin_helper.rb +42 -0
- data/lib/fluent/plugin_helper/child_process.rb +298 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
- data/lib/fluent/plugin_helper/event_loop.rb +118 -0
- data/lib/fluent/plugin_helper/formatter.rb +149 -0
- data/lib/fluent/plugin_helper/inject.rb +125 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/retry_state.rb +177 -0
- data/lib/fluent/plugin_helper/storage.rb +331 -0
- data/lib/fluent/plugin_helper/thread.rb +147 -0
- data/lib/fluent/plugin_helper/timer.rb +90 -0
- data/lib/fluent/plugin_id.rb +63 -0
- data/lib/fluent/process.rb +504 -0
- data/lib/fluent/registry.rb +99 -0
- data/lib/fluent/root_agent.rb +314 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/supervisor.rb +680 -0
- data/lib/fluent/system_config.rb +122 -0
- data/lib/fluent/test.rb +56 -0
- data/lib/fluent/test/base.rb +85 -0
- data/lib/fluent/test/driver/base.rb +179 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +125 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +52 -0
- data/lib/fluent/test/driver/output.rb +76 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +79 -0
- data/lib/fluent/test/input_test.rb +172 -0
- data/lib/fluent/test/log.rb +73 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/time.rb +175 -0
- data/lib/fluent/timezone.rb +133 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +71 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +82 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +507 -0
- data/test/config/test_configurable.rb +1194 -0
- data/test/config/test_configure_proxy.rb +386 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +403 -0
- data/test/config/test_literal_parser.rb +297 -0
- data/test/config/test_section.rb +184 -0
- data/test/config/test_system_config.rb +120 -0
- data/test/config/test_types.rb +171 -0
- data/test/helper.rb +119 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +75 -0
- data/test/plugin/test_buf_file.rb +571 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1200 -0
- data/test/plugin/test_buffer_chunk.rb +168 -0
- data/test/plugin/test_buffer_file_chunk.rb +771 -0
- data/test/plugin/test_buffer_memory_chunk.rb +265 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_filter.rb +353 -0
- data/test/plugin/test_filter_grep.rb +119 -0
- data/test/plugin/test_filter_record_transformer.rb +600 -0
- data/test/plugin/test_filter_stdout.rb +211 -0
- data/test/plugin/test_formatter_csv.rb +94 -0
- data/test/plugin/test_formatter_json.rb +30 -0
- data/test/plugin/test_formatter_ltsv.rb +52 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_in_debug_agent.rb +28 -0
- data/test/plugin/test_in_dummy.rb +188 -0
- data/test/plugin/test_in_exec.rb +133 -0
- data/test/plugin/test_in_forward.rb +635 -0
- data/test/plugin/test_in_gc_stat.rb +39 -0
- data/test/plugin/test_in_http.rb +442 -0
- data/test/plugin/test_in_monitor_agent.rb +329 -0
- data/test/plugin/test_in_object_space.rb +64 -0
- data/test/plugin/test_in_syslog.rb +205 -0
- data/test/plugin/test_in_tail.rb +1001 -0
- data/test/plugin/test_in_tcp.rb +102 -0
- data/test/plugin/test_in_udp.rb +121 -0
- data/test/plugin/test_in_unix.rb +126 -0
- data/test/plugin/test_input.rb +122 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_buffered_null.rb +79 -0
- data/test/plugin/test_out_buffered_stdout.rb +122 -0
- data/test/plugin/test_out_copy.rb +160 -0
- data/test/plugin/test_out_exec.rb +155 -0
- data/test/plugin/test_out_exec_filter.rb +262 -0
- data/test/plugin/test_out_file.rb +383 -0
- data/test/plugin/test_out_forward.rb +590 -0
- data/test/plugin/test_out_null.rb +29 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_stdout.rb +92 -0
- data/test/plugin/test_out_stream.rb +93 -0
- data/test/plugin/test_output.rb +568 -0
- data/test/plugin/test_output_as_buffered.rb +1604 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +839 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +38 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_base.rb +32 -0
- data/test/plugin/test_parser_csv.rb +104 -0
- data/test/plugin/test_parser_json.rb +107 -0
- data/test/plugin/test_parser_labeled_tsv.rb +129 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +48 -0
- data/test/plugin/test_parser_none.rb +53 -0
- data/test/plugin/test_parser_regexp.rb +277 -0
- data/test/plugin/test_parser_syslog.rb +66 -0
- data/test/plugin/test_parser_time.rb +46 -0
- data/test/plugin/test_parser_tsv.rb +121 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +8 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/test_child_process.rb +608 -0
- data/test/plugin_helper/test_compat_parameters.rb +242 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_formatter.rb +252 -0
- data/test/plugin_helper/test_inject.rb +487 -0
- data/test/plugin_helper/test_parser.rb +263 -0
- data/test/plugin_helper/test_retry_state.rb +399 -0
- data/test/plugin_helper/test_storage.rb +521 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +131 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_config.rb +179 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_event.rb +329 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +184 -0
- data/test/test_filter.rb +121 -0
- data/test/test_formatter.rb +319 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +572 -0
- data/test/test_match.rb +137 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_output.rb +214 -0
- data/test/test_plugin_classes.rb +136 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_process.rb +48 -0
- data/test/test_root_agent.rb +278 -0
- data/test/test_supervisor.rb +339 -0
- data/test/test_time_formatter.rb +186 -0
- data/test/test_unique_id.rb +47 -0
- metadata +823 -0
@@ -0,0 +1,224 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/config/types'
|
18
|
+
require 'fluent/config/element'
|
19
|
+
|
20
|
+
module Fluent
|
21
|
+
module PluginHelper
|
22
|
+
module CompatParameters
|
23
|
+
# This plugin helper is to bring old-fashioned buffer/other
|
24
|
+
# configuration parameters to v0.14 plugin API configurations.
|
25
|
+
# This helper is mainly to convert plugins from v0.12 API
|
26
|
+
# to v0.14 API safely, without breaking user deployment.
|
27
|
+
|
28
|
+
BUFFER_PARAMS = {
|
29
|
+
"buffer_type" => "@type",
|
30
|
+
"buffer_path" => "path",
|
31
|
+
"num_threads" => "flush_thread_count",
|
32
|
+
"flush_interval" => "flush_interval",
|
33
|
+
"try_flush_interval" => "flush_thread_interval",
|
34
|
+
"queued_chunk_flush_interval" => "flush_thread_burst_interval",
|
35
|
+
"disable_retry_limit" => "retry_forever",
|
36
|
+
"retry_limit" => "retry_max_times",
|
37
|
+
"max_retry_wait" => "retry_max_interval",
|
38
|
+
"buffer_chunk_limit" => "chunk_limit_size",
|
39
|
+
"buffer_queue_limit" => "queue_length_limit",
|
40
|
+
"buffer_queue_full_action" => "overflow_action",
|
41
|
+
"flush_at_shutdown" => "flush_at_shutdown",
|
42
|
+
}
|
43
|
+
|
44
|
+
BUFFER_TIME_SLICED_PARAMS = {
|
45
|
+
"time_slice_format" => nil,
|
46
|
+
"time_slice_wait" => "timekey_wait",
|
47
|
+
}
|
48
|
+
|
49
|
+
PARSER_PARAMS = {
|
50
|
+
"format" => "@type",
|
51
|
+
"time_key" => "time_key",
|
52
|
+
"time_format" => "time_format",
|
53
|
+
"delimiter" => "delimiter",
|
54
|
+
"json_parser" => "json_parser", # JSONParser
|
55
|
+
"label_delimiter" => "label_delimiter", # LabeledTSVParser
|
56
|
+
"format_firstline" => "format_firstline", # MultilineParser
|
57
|
+
"message_key" => "message_key", # NoneParser
|
58
|
+
"with_priority" => "with_priority", # SyslogParser
|
59
|
+
}
|
60
|
+
|
61
|
+
INJECT_PARAMS = {
|
62
|
+
"include_time_key" => nil,
|
63
|
+
"time_key" => "time_key",
|
64
|
+
"time_format" => "time_format",
|
65
|
+
"timezone" => "timezone",
|
66
|
+
"include_tag_key" => nil,
|
67
|
+
"tag_key" => "tag_key",
|
68
|
+
"localtime" => nil,
|
69
|
+
"utc" => nil,
|
70
|
+
}
|
71
|
+
|
72
|
+
FORMATTER_PARAMS = {
|
73
|
+
"format" => "@type",
|
74
|
+
"delimiter" => "delimiter",
|
75
|
+
"force_quotes" => "force_quotes", # CsvFormatter
|
76
|
+
"fields" => "fields", # CsvFormatter
|
77
|
+
"json_parser" => "json_parser", # JSONFormatter
|
78
|
+
"label_delimiter" => "label_delimiter", # LabeledTSVFormatter
|
79
|
+
"output_time" => "output_time", # OutFileFormatter
|
80
|
+
"output_tag" => "output_tag", # OutFileFormatter
|
81
|
+
"message_key" => "message_key", # SingleValueFormatter
|
82
|
+
"add_newline" => "add_newline", # SingleValueFormatter
|
83
|
+
"output_type" => "output_type", # StdoutFormatter
|
84
|
+
}
|
85
|
+
|
86
|
+
def compat_parameters_convert(conf, *types, **kwargs)
|
87
|
+
types.each do |type|
|
88
|
+
case type
|
89
|
+
when :buffer
|
90
|
+
compat_parameters_buffer(conf, **kwargs)
|
91
|
+
when :inject
|
92
|
+
compat_parameters_inject(conf)
|
93
|
+
when :parser
|
94
|
+
compat_parameters_parser(conf)
|
95
|
+
when :formatter
|
96
|
+
compat_parameters_formatter(conf)
|
97
|
+
else
|
98
|
+
raise "BUG: unknown compat_parameters type: #{type}"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
conf
|
103
|
+
end
|
104
|
+
|
105
|
+
def compat_parameters_buffer(conf, default_chunk_key: '')
|
106
|
+
# return immediately if <buffer> section exists, or any buffer-related parameters don't exist
|
107
|
+
return unless conf.elements('buffer').empty?
|
108
|
+
return if (BUFFER_PARAMS.keys + BUFFER_TIME_SLICED_PARAMS.keys).all?{|k| !conf.has_key?(k) }
|
109
|
+
|
110
|
+
# TODO: warn obsolete parameters if these are deprecated
|
111
|
+
buffer_params = BUFFER_PARAMS.merge(BUFFER_TIME_SLICED_PARAMS)
|
112
|
+
attr = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
|
113
|
+
if compat_key == 'buffer_queue_full_action' && value == 'exception'
|
114
|
+
'throw_exception'
|
115
|
+
else
|
116
|
+
value
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
chunk_key = default_chunk_key
|
121
|
+
|
122
|
+
if conf.has_key?('time_slice_format')
|
123
|
+
chunk_key = 'time'
|
124
|
+
attr['timekey'] = case conf['time_slice_format']
|
125
|
+
when /\%S/ then 1
|
126
|
+
when /\%M/ then 60
|
127
|
+
when /\%H/ then 3600
|
128
|
+
when /\%d/ then 86400
|
129
|
+
else
|
130
|
+
raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
|
131
|
+
end
|
132
|
+
else
|
133
|
+
if chunk_key == 'time'
|
134
|
+
attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
|
139
|
+
conf.elements << e
|
140
|
+
|
141
|
+
conf
|
142
|
+
end
|
143
|
+
|
144
|
+
def compat_parameters_inject(conf)
|
145
|
+
return unless conf.elements('inject').empty?
|
146
|
+
return if INJECT_PARAMS.keys.all?{|k| !conf.has_key?(k) }
|
147
|
+
|
148
|
+
# TODO: warn obsolete parameters if these are deprecated
|
149
|
+
attr = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)
|
150
|
+
|
151
|
+
if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
|
152
|
+
attr['time_key'] ||= 'time'
|
153
|
+
attr['time_type'] ||= 'string'
|
154
|
+
end
|
155
|
+
if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
|
156
|
+
attr['time_type'] = 'unixtime'
|
157
|
+
end
|
158
|
+
if conf.has_key?('localtime') || conf.has_key?('utc')
|
159
|
+
if conf.has_key?('localtime') && conf.has_key?('utc')
|
160
|
+
raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
|
161
|
+
elsif conf.has_key?('localtime')
|
162
|
+
attr['localtime'] = Fluent::Config.bool_value(conf['localtime'])
|
163
|
+
elsif conf.has_key?('utc')
|
164
|
+
attr['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
|
165
|
+
# Specifying "localtime false" means using UTC in TimeFormatter
|
166
|
+
# And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
|
167
|
+
# There are difference between "Z" and "+0000" in timezone formatting.
|
168
|
+
# TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
|
173
|
+
attr['tag_key'] ||= 'tag'
|
174
|
+
end
|
175
|
+
|
176
|
+
e = Fluent::Config::Element.new('inject', '', attr, [])
|
177
|
+
conf.elements << e
|
178
|
+
|
179
|
+
conf
|
180
|
+
end
|
181
|
+
|
182
|
+
def compat_parameters_parser(conf)
|
183
|
+
return unless conf.elements('parse').empty?
|
184
|
+
return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
|
185
|
+
|
186
|
+
# TODO: warn obsolete parameters if these are deprecated
|
187
|
+
attr = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
|
188
|
+
|
189
|
+
e = Fluent::Config::Element.new('parse', '', attr, [])
|
190
|
+
conf.elements << e
|
191
|
+
|
192
|
+
conf
|
193
|
+
end
|
194
|
+
|
195
|
+
def compat_parameters_formatter(conf)
|
196
|
+
return unless conf.elements('format').empty?
|
197
|
+
return if FORMATTER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
|
198
|
+
|
199
|
+
# TODO: warn obsolete parameters if these are deprecated
|
200
|
+
attr = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)
|
201
|
+
|
202
|
+
e = Fluent::Config::Element.new('format', '', attr, [])
|
203
|
+
conf.elements << e
|
204
|
+
|
205
|
+
conf
|
206
|
+
end
|
207
|
+
|
208
|
+
def compat_parameters_copy_to_subsection_attributes(conf, params, &block)
|
209
|
+
attr = {}
|
210
|
+
params.each do |compat, current|
|
211
|
+
next unless current
|
212
|
+
if conf.has_key?(compat)
|
213
|
+
if block_given?
|
214
|
+
attr[current] = block.call(compat, conf[compat])
|
215
|
+
else
|
216
|
+
attr[current] = conf[compat]
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
220
|
+
attr
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/time'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module PluginHelper
|
21
|
+
module EventEmitter
|
22
|
+
# stop : [-]
|
23
|
+
# shutdown : disable @router
|
24
|
+
# close : [-]
|
25
|
+
# terminate: [-]
|
26
|
+
|
27
|
+
def router
|
28
|
+
@_event_emitter_used_actually = true
|
29
|
+
@router
|
30
|
+
end
|
31
|
+
|
32
|
+
def router=(r)
|
33
|
+
@router = r
|
34
|
+
end
|
35
|
+
|
36
|
+
def has_router?
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
def event_emitter_used_actually?
|
41
|
+
@_event_emitter_used_actually
|
42
|
+
end
|
43
|
+
|
44
|
+
def event_emitter_router(label_name)
|
45
|
+
if label_name
|
46
|
+
Engine.root_agent.find_label(label_name).event_router
|
47
|
+
else
|
48
|
+
Engine.root_agent.event_router
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def initialize
|
53
|
+
super
|
54
|
+
@_event_emitter_used_actually = false
|
55
|
+
@router = nil
|
56
|
+
end
|
57
|
+
|
58
|
+
def configure(conf)
|
59
|
+
require 'fluent/engine'
|
60
|
+
super
|
61
|
+
@router = event_emitter_router(conf['@label'])
|
62
|
+
end
|
63
|
+
|
64
|
+
def after_shutdown
|
65
|
+
@router = nil
|
66
|
+
super
|
67
|
+
end
|
68
|
+
|
69
|
+
def close # unset router many times to reduce test cost
|
70
|
+
@router = nil
|
71
|
+
super
|
72
|
+
end
|
73
|
+
|
74
|
+
def terminate
|
75
|
+
@router = nil
|
76
|
+
super
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'cool.io'
|
18
|
+
require 'fluent/plugin_helper/thread'
|
19
|
+
|
20
|
+
module Fluent
|
21
|
+
module PluginHelper
|
22
|
+
module EventLoop
|
23
|
+
# Currently this plugin helper is only for other helpers, not plugins.
|
24
|
+
# there's no way to create customized watchers to attach event loops.
|
25
|
+
include Fluent::PluginHelper::Thread
|
26
|
+
|
27
|
+
# stop : [-]
|
28
|
+
# shutdown : detach all event watchers on event loop
|
29
|
+
# close : stop event loop
|
30
|
+
# terminate: initialize internal state
|
31
|
+
|
32
|
+
EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
|
33
|
+
|
34
|
+
attr_reader :_event_loop # for tests
|
35
|
+
|
36
|
+
def event_loop_attach(watcher)
|
37
|
+
@_event_loop_mutex.synchronize do
|
38
|
+
@_event_loop.attach(watcher)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def event_loop_wait_until_start
|
43
|
+
sleep(0.1) until event_loop_running?
|
44
|
+
end
|
45
|
+
|
46
|
+
def event_loop_wait_until_stop
|
47
|
+
sleep(0.1) while event_loop_running?
|
48
|
+
end
|
49
|
+
|
50
|
+
def event_loop_running?
|
51
|
+
@_event_loop_running
|
52
|
+
end
|
53
|
+
|
54
|
+
def initialize
|
55
|
+
super
|
56
|
+
@_event_loop = Coolio::Loop.new
|
57
|
+
@_event_loop_running = false
|
58
|
+
@_event_loop_mutex = Mutex.new
|
59
|
+
# plugin MAY configure loop run timeout in #configure
|
60
|
+
@_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
|
61
|
+
end
|
62
|
+
|
63
|
+
def start
|
64
|
+
super
|
65
|
+
|
66
|
+
# event loop does not run here, so mutex lock is not required
|
67
|
+
thread_create :event_loop do
|
68
|
+
default_watcher = DefaultWatcher.new
|
69
|
+
@_event_loop.attach(default_watcher)
|
70
|
+
@_event_loop_running = true
|
71
|
+
@_event_loop.run(@_event_loop_run_timeout) # this method blocks
|
72
|
+
@_event_loop_running = false
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def shutdown
|
77
|
+
@_event_loop_mutex.synchronize do
|
78
|
+
@_event_loop.watchers.each {|w| w.detach if w.attached? }
|
79
|
+
end
|
80
|
+
while @_event_loop_running
|
81
|
+
sleep 0.1
|
82
|
+
end
|
83
|
+
|
84
|
+
super
|
85
|
+
end
|
86
|
+
|
87
|
+
def close
|
88
|
+
if @_event_loop_running
|
89
|
+
begin
|
90
|
+
@_event_loop.stop # we cannot check loop is running or not
|
91
|
+
rescue RuntimeError => e
|
92
|
+
raise unless e.message == 'loop not running'
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
super
|
97
|
+
end
|
98
|
+
|
99
|
+
def terminate
|
100
|
+
@_event_loop = nil
|
101
|
+
@_event_loop_running = false
|
102
|
+
@_event_loop_mutex = nil
|
103
|
+
@_event_loop_run_timeout = nil
|
104
|
+
|
105
|
+
super
|
106
|
+
end
|
107
|
+
|
108
|
+
# watcher to block to run event loop until shutdown
|
109
|
+
class DefaultWatcher < Coolio::TimerWatcher
|
110
|
+
def initialize
|
111
|
+
super(1, true) # interval: 1, repeat: true
|
112
|
+
end
|
113
|
+
# do nothing
|
114
|
+
def on_timer; end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,149 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin'
|
18
|
+
require 'fluent/plugin/formatter'
|
19
|
+
require 'fluent/config/element'
|
20
|
+
require 'fluent/configurable'
|
21
|
+
|
22
|
+
module Fluent
|
23
|
+
module PluginHelper
|
24
|
+
module Formatter
|
25
|
+
def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)
|
26
|
+
formatter = @_formatters[usage]
|
27
|
+
return formatter if formatter
|
28
|
+
|
29
|
+
type = if type
|
30
|
+
type
|
31
|
+
elsif conf && conf.respond_to?(:[])
|
32
|
+
raise Fluent::ConfigError, "@type is required in <format>" unless conf['@type']
|
33
|
+
conf['@type']
|
34
|
+
elsif default_type
|
35
|
+
default_type
|
36
|
+
else
|
37
|
+
raise ArgumentError, "BUG: both type and conf are not specified"
|
38
|
+
end
|
39
|
+
formatter = Fluent::Plugin.new_formatter(type, parent: self)
|
40
|
+
config = case conf
|
41
|
+
when Fluent::Config::Element
|
42
|
+
conf
|
43
|
+
when Hash
|
44
|
+
# in code, programmer may use symbols as keys, but Element needs strings
|
45
|
+
conf = Hash[conf.map{|k,v| [k.to_s, v]}]
|
46
|
+
Fluent::Config::Element.new('format', usage, conf, [])
|
47
|
+
when nil
|
48
|
+
Fluent::Config::Element.new('format', usage, {}, [])
|
49
|
+
else
|
50
|
+
raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
|
51
|
+
end
|
52
|
+
formatter.configure(config)
|
53
|
+
if @_formatters_started
|
54
|
+
formatter.start
|
55
|
+
end
|
56
|
+
|
57
|
+
@_formatters[usage] = formatter
|
58
|
+
formatter
|
59
|
+
end
|
60
|
+
|
61
|
+
module FormatterParams
|
62
|
+
include Fluent::Configurable
|
63
|
+
# minimum section definition to instantiate formatter plugin instances
|
64
|
+
config_section :format, required: false, multi: true, param_name: :formatter_configs do
|
65
|
+
config_argument :usage, :string, default: ''
|
66
|
+
config_param :@type, :string
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.included(mod)
|
71
|
+
mod.include FormatterParams
|
72
|
+
end
|
73
|
+
|
74
|
+
attr_reader :_formatters # for tests
|
75
|
+
|
76
|
+
def initialize
|
77
|
+
super
|
78
|
+
@_formatters_started = false
|
79
|
+
@_formatters = {} # usage => formatter
|
80
|
+
end
|
81
|
+
|
82
|
+
def configure(conf)
|
83
|
+
super
|
84
|
+
|
85
|
+
if @formatter_configs
|
86
|
+
@formatter_configs.each do |section|
|
87
|
+
if @_formatters[section.usage]
|
88
|
+
raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
|
89
|
+
end
|
90
|
+
formatter = Plugin.new_formatter(section[:@type], parent: self)
|
91
|
+
formatter.configure(section.corresponding_config_element)
|
92
|
+
@_formatters[section.usage] = formatter
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def start
|
98
|
+
super
|
99
|
+
@_formatters_started = true
|
100
|
+
@_formatters.each_pair do |usage, formatter|
|
101
|
+
formatter.start
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def formatter_operate(method_name, &block)
|
106
|
+
@_formatters.each_pair do |usage, formatter|
|
107
|
+
begin
|
108
|
+
formatter.send(method_name)
|
109
|
+
block.call(formatter) if block_given?
|
110
|
+
rescue => e
|
111
|
+
log.error "unexpected error while #{method_name}", usage: usage, formatter: formatter, error: e
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def stop
|
117
|
+
super
|
118
|
+
formatter_operate(:stop)
|
119
|
+
end
|
120
|
+
|
121
|
+
def before_shutdown
|
122
|
+
formatter_operate(:before_shutdown)
|
123
|
+
super
|
124
|
+
end
|
125
|
+
|
126
|
+
def shutdown
|
127
|
+
formatter_operate(:shutdown)
|
128
|
+
super
|
129
|
+
end
|
130
|
+
|
131
|
+
def after_shutdown
|
132
|
+
formatter_operate(:after_shutdown)
|
133
|
+
super
|
134
|
+
end
|
135
|
+
|
136
|
+
def close
|
137
|
+
formatter_operate(:close)
|
138
|
+
super
|
139
|
+
end
|
140
|
+
|
141
|
+
def terminate
|
142
|
+
formatter_operate(:terminate)
|
143
|
+
@_formatters_started = false
|
144
|
+
@_formatters = {}
|
145
|
+
super
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|