fluentd 1.20.0-aarch64-mingw-ucrt
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rubocop.yml +116 -0
- data/ADOPTERS.md +5 -0
- data/AGENTS.md +46 -0
- data/AUTHORS +2 -0
- data/CHANGELOG.md +3324 -0
- data/CONTRIBUTING.md +45 -0
- data/GOVERNANCE.md +55 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +19 -0
- data/README.md +73 -0
- data/Rakefile +89 -0
- data/SECURITY.md +47 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-cat +5 -0
- data/bin/fluent-ctl +7 -0
- data/bin/fluent-debug +5 -0
- data/bin/fluent-gem +9 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +15 -0
- data/code-of-conduct.md +3 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +14 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +16 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_sample_blocks.conf +17 -0
- data/example/in_sample_with_compression.conf +23 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_copy.conf +20 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_sd.conf +17 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/sd.yaml +8 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +139 -0
- data/lib/fluent/agent.rb +168 -0
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/clock.rb +66 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/ca_generate.rb +184 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/cat.rb +365 -0
- data/lib/fluent/command/ctl.rb +180 -0
- data/lib/fluent/command/debug.rb +103 -0
- data/lib/fluent/command/fluentd.rb +385 -0
- data/lib/fluent/command/plugin_config_formatter.rb +308 -0
- data/lib/fluent/command/plugin_generator.rb +365 -0
- data/lib/fluent/compat/call_super_mixin.rb +76 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +144 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +117 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +721 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/basic_parser.rb +122 -0
- data/lib/fluent/config/configure_proxy.rb +424 -0
- data/lib/fluent/config/dsl.rb +152 -0
- data/lib/fluent/config/element.rb +265 -0
- data/lib/fluent/config/error.rb +44 -0
- data/lib/fluent/config/literal_parser.rb +305 -0
- data/lib/fluent/config/parser.rb +119 -0
- data/lib/fluent/config/section.rb +272 -0
- data/lib/fluent/config/types.rb +257 -0
- data/lib/fluent/config/v1_parser.rb +200 -0
- data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
- data/lib/fluent/config/yaml_parser/loader.rb +124 -0
- data/lib/fluent/config/yaml_parser/parser.rb +179 -0
- data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
- data/lib/fluent/config/yaml_parser.rb +56 -0
- data/lib/fluent/config.rb +89 -0
- data/lib/fluent/configurable.rb +202 -0
- data/lib/fluent/counter/base_socket.rb +44 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +14 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/engine.rb +269 -0
- data/lib/fluent/env.rb +45 -0
- data/lib/fluent/error.rb +37 -0
- data/lib/fluent/event.rb +369 -0
- data/lib/fluent/event_router.rb +315 -0
- data/lib/fluent/ext_monitor_require.rb +28 -0
- data/lib/fluent/file_wrapper.rb +137 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/fluent_log_event_router.rb +139 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +46 -0
- data/lib/fluent/load.rb +33 -0
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +775 -0
- data/lib/fluent/match.rb +187 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +111 -0
- data/lib/fluent/oj_options.rb +61 -0
- data/lib/fluent/output.rb +29 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +88 -0
- data/lib/fluent/plugin/base.rb +219 -0
- data/lib/fluent/plugin/buf_file.rb +256 -0
- data/lib/fluent/plugin/buf_file_single.rb +272 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer/chunk.rb +313 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +432 -0
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +325 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
- data/lib/fluent/plugin/buffer.rb +1063 -0
- data/lib/fluent/plugin/compressable.rb +87 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/extractor.rb +132 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/filter.rb +119 -0
- data/lib/fluent/plugin/filter_grep.rb +189 -0
- data/lib/fluent/plugin/filter_parser.rb +124 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +325 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +75 -0
- data/lib/fluent/plugin/formatter_csv.rb +92 -0
- data/lib/fluent/plugin/formatter_hash.rb +35 -0
- data/lib/fluent/plugin/formatter_json.rb +62 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +55 -0
- data/lib/fluent/plugin/formatter_single_value.rb +36 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +40 -0
- data/lib/fluent/plugin/in_debug_agent.rb +71 -0
- data/lib/fluent/plugin/in_dummy.rb +18 -0
- data/lib/fluent/plugin/in_exec.rb +124 -0
- data/lib/fluent/plugin/in_forward.rb +489 -0
- data/lib/fluent/plugin/in_gc_stat.rb +72 -0
- data/lib/fluent/plugin/in_http.rb +737 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +406 -0
- data/lib/fluent/plugin/in_object_space.rb +93 -0
- data/lib/fluent/plugin/in_sample.rb +148 -0
- data/lib/fluent/plugin/in_syslog.rb +346 -0
- data/lib/fluent/plugin/in_tail/compatibility.rb +111 -0
- data/lib/fluent/plugin/in_tail/dir_watcher.rb +713 -0
- data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
- data/lib/fluent/plugin/in_tail/io_handler.rb +391 -0
- data/lib/fluent/plugin/in_tail/line_feeder.rb +230 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
- data/lib/fluent/plugin/in_tail/stat_watcher.rb +52 -0
- data/lib/fluent/plugin/in_tail/tail_watcher.rb +276 -0
- data/lib/fluent/plugin/in_tail/worker_pool.rb +298 -0
- data/lib/fluent/plugin/in_tail.rb +917 -0
- data/lib/fluent/plugin/in_tcp.rb +230 -0
- data/lib/fluent/plugin/in_udp.rb +101 -0
- data/lib/fluent/plugin/in_unix.rb +197 -0
- data/lib/fluent/plugin/input.rb +71 -0
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +179 -0
- data/lib/fluent/plugin/out_buffer.rb +40 -0
- data/lib/fluent/plugin/out_copy.rb +120 -0
- data/lib/fluent/plugin/out_exec.rb +105 -0
- data/lib/fluent/plugin/out_exec_filter.rb +319 -0
- data/lib/fluent/plugin/out_file.rb +371 -0
- data/lib/fluent/plugin/out_forward/ack_handler.rb +176 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +131 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +177 -0
- data/lib/fluent/plugin/out_forward.rb +865 -0
- data/lib/fluent/plugin/out_http.rb +419 -0
- data/lib/fluent/plugin/out_null.rb +74 -0
- data/lib/fluent/plugin/out_relabel.rb +32 -0
- data/lib/fluent/plugin/out_roundrobin.rb +84 -0
- data/lib/fluent/plugin/out_secondary_file.rb +148 -0
- data/lib/fluent/plugin/out_stdout.rb +81 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1688 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +41 -0
- data/lib/fluent/plugin/parser.rb +274 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +124 -0
- data/lib/fluent/plugin/parser_json.rb +118 -0
- data/lib/fluent/plugin/parser_ltsv.rb +49 -0
- data/lib/fluent/plugin/parser_msgpack.rb +72 -0
- data/lib/fluent/plugin/parser_multiline.rb +152 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +543 -0
- data/lib/fluent/plugin/parser_tsv.rb +47 -0
- data/lib/fluent/plugin/sd_file.rb +157 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin/sd_static.rb +58 -0
- data/lib/fluent/plugin/service_discovery.rb +65 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +166 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin.rb +206 -0
- data/lib/fluent/plugin_helper/cert_option.rb +199 -0
- data/lib/fluent/plugin_helper/child_process.rb +369 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +112 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +81 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +53 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +116 -0
- data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
- data/lib/fluent/plugin_helper/http_server.rb +129 -0
- data/lib/fluent/plugin_helper/inject.rb +154 -0
- data/lib/fluent/plugin_helper/metrics.rb +136 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
- data/lib/fluent/plugin_helper/retry_state.rb +219 -0
- data/lib/fluent/plugin_helper/server.rb +840 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
- data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
- data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
- data/lib/fluent/plugin_helper/socket.rb +288 -0
- data/lib/fluent/plugin_helper/socket_option.rb +98 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +180 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +75 -0
- data/lib/fluent/plugin_id.rb +93 -0
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/registry.rb +117 -0
- data/lib/fluent/root_agent.rb +468 -0
- data/lib/fluent/rpc.rb +95 -0
- data/lib/fluent/source_only_buffer_agent.rb +102 -0
- data/lib/fluent/static_config_analysis.rb +195 -0
- data/lib/fluent/supervisor.rb +1347 -0
- data/lib/fluent/system_config.rb +213 -0
- data/lib/fluent/test/base.rb +83 -0
- data/lib/fluent/test/driver/base.rb +231 -0
- data/lib/fluent/test/driver/base_owned.rb +83 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +61 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/storage.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +138 -0
- data/lib/fluent/test/input_test.rb +174 -0
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/test/startup_shutdown.rb +44 -0
- data/lib/fluent/test.rb +61 -0
- data/lib/fluent/time.rb +513 -0
- data/lib/fluent/timezone.rb +171 -0
- data/lib/fluent/tls.rb +105 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/variable_store.rb +40 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/win32api.rb +38 -0
- data/lib/fluent/winsvc.rb +130 -0
- data/templates/new_gem/.gitignore +8 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +28 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
- data/templates/new_gem/test/helper.rb.erb +7 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md-table.erb +10 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- metadata +996 -0
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Fluent
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
require 'set'
|
|
18
|
+
require 'fluent/env'
|
|
19
|
+
require 'fluent/variable_store'
|
|
20
|
+
|
|
21
|
+
module Fluent
|
|
22
|
+
module PluginId
|
|
23
|
+
|
|
24
|
+
def initialize
|
|
25
|
+
super
|
|
26
|
+
|
|
27
|
+
@_plugin_id_variable_store = nil
|
|
28
|
+
@_plugin_root_dir = nil
|
|
29
|
+
@id = nil
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def configure(conf)
|
|
33
|
+
@_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:plugin_id, default_value: Set.new)
|
|
34
|
+
@id = conf['@id']
|
|
35
|
+
@_id_configured = !!@id # plugin id is explicitly configured by users (or not)
|
|
36
|
+
if @id
|
|
37
|
+
@id = @id.to_s
|
|
38
|
+
if @_plugin_id_variable_store.include?(@id) && !plugin_id_for_test?
|
|
39
|
+
raise Fluent::ConfigError, "Duplicated plugin id `#{@id}`. Check whole configuration and fix it."
|
|
40
|
+
end
|
|
41
|
+
@_plugin_id_variable_store.add(@id)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
super
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def plugin_id_for_test?
|
|
48
|
+
caller_locations.each do |location|
|
|
49
|
+
# Thread::Backtrace::Location#path returns base filename or absolute path.
|
|
50
|
+
# #absolute_path returns absolute_path always.
|
|
51
|
+
# https://bugs.ruby-lang.org/issues/12159
|
|
52
|
+
if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
|
|
53
|
+
return true
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
false
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def plugin_id_configured?
|
|
60
|
+
if instance_variable_defined?(:@_id_configured)
|
|
61
|
+
@_id_configured
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def plugin_id
|
|
66
|
+
if instance_variable_defined?(:@id)
|
|
67
|
+
@id || "object:#{object_id.to_s(16)}"
|
|
68
|
+
else
|
|
69
|
+
"object:#{object_id.to_s(16)}"
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def plugin_root_dir
|
|
74
|
+
return @_plugin_root_dir if @_plugin_root_dir
|
|
75
|
+
return nil unless system_config.root_dir
|
|
76
|
+
return nil unless plugin_id_configured?
|
|
77
|
+
|
|
78
|
+
# Fluent::Plugin::Base#fluentd_worker_id
|
|
79
|
+
dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
|
|
80
|
+
FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
|
|
81
|
+
@_plugin_root_dir = dir.freeze
|
|
82
|
+
dir
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def stop
|
|
86
|
+
if @_plugin_id_variable_store
|
|
87
|
+
@_plugin_id_variable_store.delete(@id)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
super
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Fluentd
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
require 'fluent/compat/detach_process_mixin'
|
|
18
|
+
|
|
19
|
+
module Fluent
|
|
20
|
+
DetachProcessMixin = Fluent::Compat::DetachProcessMixin
|
|
21
|
+
DetachMultiProcessMixin = Fluent::Compat::DetachMultiProcessMixin
|
|
22
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Fluentd
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
require 'fluent/config/error'
|
|
18
|
+
|
|
19
|
+
module Fluent
|
|
20
|
+
class Registry
|
|
21
|
+
DEFAULT_PLUGIN_PATH = File.expand_path('../plugin', __FILE__)
|
|
22
|
+
FLUENT_LIB_PATH = File.dirname(File.dirname(DEFAULT_PLUGIN_PATH))
|
|
23
|
+
|
|
24
|
+
def initialize(kind, search_prefix, dir_search_prefix: nil)
|
|
25
|
+
@kind = kind
|
|
26
|
+
@search_prefix = search_prefix
|
|
27
|
+
@dir_search_prefix = dir_search_prefix
|
|
28
|
+
@map = {}
|
|
29
|
+
@paths = []
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
attr_reader :kind, :paths, :map, :dir_search_prefix
|
|
33
|
+
|
|
34
|
+
def register(type, value)
|
|
35
|
+
type = type.to_sym
|
|
36
|
+
@map[type] = value
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def lookup(type)
|
|
40
|
+
type = type.to_sym
|
|
41
|
+
if value = @map[type]
|
|
42
|
+
return value
|
|
43
|
+
end
|
|
44
|
+
search(type)
|
|
45
|
+
if value = @map[type]
|
|
46
|
+
return value
|
|
47
|
+
end
|
|
48
|
+
raise NotFoundPluginError.new("Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins",
|
|
49
|
+
kind: @kind, type: type)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def reverse_lookup(value)
|
|
53
|
+
@map.each do |k, v|
|
|
54
|
+
return k if v == value
|
|
55
|
+
end
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def search(type)
|
|
60
|
+
# search from additional plugin directories
|
|
61
|
+
if @dir_search_prefix
|
|
62
|
+
path = "#{@dir_search_prefix}#{type}"
|
|
63
|
+
files = @paths.filter_map { |lp|
|
|
64
|
+
lpath = File.expand_path(File.join(lp, "#{path}.rb"))
|
|
65
|
+
File.exist?(lpath) ? lpath : nil
|
|
66
|
+
}
|
|
67
|
+
unless files.empty?
|
|
68
|
+
# prefer newer version
|
|
69
|
+
require files.max
|
|
70
|
+
return
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
path = "#{@search_prefix}#{type}"
|
|
75
|
+
|
|
76
|
+
# prefer LOAD_PATH than gems
|
|
77
|
+
files = $LOAD_PATH.filter_map { |lp|
|
|
78
|
+
if lp == FLUENT_LIB_PATH
|
|
79
|
+
nil
|
|
80
|
+
else
|
|
81
|
+
lpath = File.expand_path(File.join(lp, "#{path}.rb"))
|
|
82
|
+
File.exist?(lpath) ? lpath : nil
|
|
83
|
+
end
|
|
84
|
+
}
|
|
85
|
+
unless files.empty?
|
|
86
|
+
# prefer newer version
|
|
87
|
+
require files.max
|
|
88
|
+
return
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Find from gems and prefer newer version
|
|
92
|
+
specs = Gem::Specification.find_all { |spec|
|
|
93
|
+
if spec.name == 'fluentd'.freeze
|
|
94
|
+
false
|
|
95
|
+
else
|
|
96
|
+
spec.contains_requirable_file? path
|
|
97
|
+
end
|
|
98
|
+
}.sort_by { |spec| spec.version }
|
|
99
|
+
if spec = specs.last
|
|
100
|
+
spec.require_paths.each { |lib|
|
|
101
|
+
file = "#{spec.full_gem_path}/#{lib}/#{path}"
|
|
102
|
+
if File.exist?("#{file}.rb")
|
|
103
|
+
require file
|
|
104
|
+
return
|
|
105
|
+
end
|
|
106
|
+
}
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Lastly, load built-in plugin
|
|
110
|
+
lpath = File.expand_path(File.join(FLUENT_LIB_PATH, "#{@search_prefix}#{type}.rb"))
|
|
111
|
+
if File.exist?(lpath)
|
|
112
|
+
require lpath
|
|
113
|
+
return
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Fluentd
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
require 'delegate'
|
|
18
|
+
|
|
19
|
+
require 'fluent/config/error'
|
|
20
|
+
require 'fluent/agent'
|
|
21
|
+
require 'fluent/label'
|
|
22
|
+
require 'fluent/plugin'
|
|
23
|
+
require 'fluent/system_config'
|
|
24
|
+
require 'fluent/time'
|
|
25
|
+
require 'fluent/source_only_buffer_agent'
|
|
26
|
+
|
|
27
|
+
module Fluent
|
|
28
|
+
#
|
|
29
|
+
# Fluentd forms a tree structure to manage plugins:
|
|
30
|
+
#
|
|
31
|
+
# RootAgent
|
|
32
|
+
# |
|
|
33
|
+
# +------------+-------------+-------------+
|
|
34
|
+
# | | | |
|
|
35
|
+
# <label> <source> <filter> <match>
|
|
36
|
+
# |
|
|
37
|
+
# +----+----+
|
|
38
|
+
# | |
|
|
39
|
+
# <filter> <match>
|
|
40
|
+
#
|
|
41
|
+
# Relation:
|
|
42
|
+
# * RootAgent has many <label>, <source>, <filter> and <match>
|
|
43
|
+
# * <label> has many <match> and <filter>
|
|
44
|
+
#
|
|
45
|
+
# Next step: `fluentd/agent.rb`
|
|
46
|
+
# Next step: 'fluentd/label.rb'
|
|
47
|
+
#
|
|
48
|
+
class RootAgent < Agent
|
|
49
|
+
ERROR_LABEL = "@ERROR".freeze # @ERROR is built-in error label
|
|
50
|
+
|
|
51
|
+
class SourceOnlyMode
|
|
52
|
+
DISABLED = 0
|
|
53
|
+
NORMAL = 1
|
|
54
|
+
ONLY_ZERO_DOWNTIME_RESTART_READY = 2
|
|
55
|
+
|
|
56
|
+
def initialize(with_source_only, start_in_parallel)
|
|
57
|
+
if start_in_parallel
|
|
58
|
+
@mode = ONLY_ZERO_DOWNTIME_RESTART_READY
|
|
59
|
+
elsif with_source_only
|
|
60
|
+
@mode = NORMAL
|
|
61
|
+
else
|
|
62
|
+
@mode = DISABLED
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def enabled?
|
|
67
|
+
@mode != DISABLED
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def only_zero_downtime_restart_ready?
|
|
71
|
+
@mode == ONLY_ZERO_DOWNTIME_RESTART_READY
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def disable!
|
|
75
|
+
@mode = DISABLED
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def initialize(log:, system_config: SystemConfig.new, start_in_parallel: false)
|
|
80
|
+
super(log: log)
|
|
81
|
+
|
|
82
|
+
@labels = {}
|
|
83
|
+
@inputs = []
|
|
84
|
+
@suppress_emit_error_log_interval = 0
|
|
85
|
+
@next_emit_error_log_time = nil
|
|
86
|
+
@without_source = system_config.without_source || false
|
|
87
|
+
@source_only_mode = SourceOnlyMode.new(system_config.with_source_only, start_in_parallel)
|
|
88
|
+
@source_only_buffer_agent = nil
|
|
89
|
+
@enable_input_metrics = system_config.enable_input_metrics
|
|
90
|
+
|
|
91
|
+
suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
attr_reader :inputs
|
|
95
|
+
attr_reader :labels
|
|
96
|
+
|
|
97
|
+
def source_only_router
|
|
98
|
+
raise "[BUG] 'RootAgent#source_only_router' should not be called when 'with_source_only' is false" unless @source_only_mode.enabled?
|
|
99
|
+
@source_only_buffer_agent.event_router
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def configure(conf)
|
|
103
|
+
used_worker_ids = []
|
|
104
|
+
available_worker_ids = (0..Fluent::Engine.system_config.workers - 1).to_a
|
|
105
|
+
# initialize <worker> elements
|
|
106
|
+
supported_directives = ['source', 'match', 'filter', 'label']
|
|
107
|
+
conf.elements(name: 'worker').each do |e|
|
|
108
|
+
target_worker_id_str = e.arg
|
|
109
|
+
if target_worker_id_str.empty?
|
|
110
|
+
raise Fluent::ConfigError, "Missing worker id on <worker> directive"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
target_worker_ids = target_worker_id_str.split("-")
|
|
114
|
+
if target_worker_ids.size == 2
|
|
115
|
+
first_worker_id = target_worker_ids.first.to_i
|
|
116
|
+
last_worker_id = target_worker_ids.last.to_i
|
|
117
|
+
if first_worker_id > last_worker_id
|
|
118
|
+
raise Fluent::ConfigError, "greater first_worker_id<#{first_worker_id}> than last_worker_id<#{last_worker_id}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"
|
|
119
|
+
end
|
|
120
|
+
target_worker_ids = []
|
|
121
|
+
first_worker_id.step(last_worker_id, 1) do |worker_id|
|
|
122
|
+
target_worker_id = worker_id.to_i
|
|
123
|
+
target_worker_ids << target_worker_id
|
|
124
|
+
|
|
125
|
+
if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
|
|
126
|
+
raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
|
|
127
|
+
end
|
|
128
|
+
available_worker_ids.delete(target_worker_id) if available_worker_ids.include?(target_worker_id)
|
|
129
|
+
if used_worker_ids.include?(target_worker_id)
|
|
130
|
+
raise Fluent::ConfigError, "specified worker_id<#{worker_id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
|
|
131
|
+
end
|
|
132
|
+
used_worker_ids << target_worker_id
|
|
133
|
+
|
|
134
|
+
e.elements.each do |elem|
|
|
135
|
+
unless supported_directives.include?(elem.name)
|
|
136
|
+
raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
unless target_worker_ids.empty?
|
|
141
|
+
e.set_target_worker_ids(target_worker_ids.uniq)
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
else
|
|
145
|
+
target_worker_id = target_worker_id_str.to_i
|
|
146
|
+
if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
|
|
147
|
+
raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
e.elements.each do |elem|
|
|
151
|
+
unless supported_directives.include?(elem.name)
|
|
152
|
+
raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
|
|
153
|
+
end
|
|
154
|
+
elem.set_target_worker_id(target_worker_id)
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
conf += e
|
|
158
|
+
end
|
|
159
|
+
conf.elements.delete_if{|e| e.name == 'worker'}
|
|
160
|
+
|
|
161
|
+
error_label_config = nil
|
|
162
|
+
|
|
163
|
+
# initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
|
|
164
|
+
label_configs = {}
|
|
165
|
+
conf.elements(name: 'label').each { |e|
|
|
166
|
+
if !Fluent::Engine.supervisor_mode && e.for_another_worker?
|
|
167
|
+
next
|
|
168
|
+
end
|
|
169
|
+
name = e.arg
|
|
170
|
+
raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
|
|
171
|
+
raise ConfigError, "@ROOT for <label> is not permitted, reserved for getting root router" if name == '@ROOT'
|
|
172
|
+
|
|
173
|
+
if name == ERROR_LABEL
|
|
174
|
+
error_label_config = e
|
|
175
|
+
else
|
|
176
|
+
add_label(name)
|
|
177
|
+
label_configs[name] = e
|
|
178
|
+
end
|
|
179
|
+
}
|
|
180
|
+
# Call 'configure' here to avoid 'label not found'
|
|
181
|
+
label_configs.each { |name, e| @labels[name].configure(e) }
|
|
182
|
+
setup_error_label(error_label_config) if error_label_config
|
|
183
|
+
|
|
184
|
+
super
|
|
185
|
+
|
|
186
|
+
setup_source_only_buffer_agent if @source_only_mode.enabled?
|
|
187
|
+
|
|
188
|
+
# initialize <source> elements
|
|
189
|
+
if @without_source
|
|
190
|
+
log.info :worker0, "'--without-source' is applied. Ignore <source> sections"
|
|
191
|
+
else
|
|
192
|
+
conf.elements(name: 'source').each { |e|
|
|
193
|
+
if !Fluent::Engine.supervisor_mode && e.for_another_worker?
|
|
194
|
+
next
|
|
195
|
+
end
|
|
196
|
+
type = e['@type']
|
|
197
|
+
raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
|
|
198
|
+
add_source(type, e)
|
|
199
|
+
}
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def setup_error_label(e)
|
|
204
|
+
error_label = add_label(ERROR_LABEL)
|
|
205
|
+
error_label.configure(e)
|
|
206
|
+
@error_collector = error_label.event_router
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def setup_source_only_buffer_agent(flush: false)
|
|
210
|
+
@source_only_buffer_agent = SourceOnlyBufferAgent.new(log: log, system_config: Fluent::Engine.system_config)
|
|
211
|
+
@source_only_buffer_agent.configure(flush: flush)
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def cleanup_source_only_buffer_agent
|
|
215
|
+
@source_only_buffer_agent&.cleanup
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def lifecycle(desc: false, kind_callback: nil, kind_or_agent_list: nil)
|
|
219
|
+
only_zero_downtime_restart_ready = false
|
|
220
|
+
|
|
221
|
+
unless kind_or_agent_list
|
|
222
|
+
if @source_only_mode.enabled?
|
|
223
|
+
kind_or_agent_list = [:input, @source_only_buffer_agent]
|
|
224
|
+
only_zero_downtime_restart_ready = @source_only_mode.only_zero_downtime_restart_ready?
|
|
225
|
+
elsif @source_only_buffer_agent
|
|
226
|
+
# source_only_buffer_agent can re-reroute events, so the priority is equal to output_with_router.
|
|
227
|
+
kind_or_agent_list = [:input, :output_with_router, @source_only_buffer_agent, @labels.values, :filter, :output].flatten
|
|
228
|
+
else
|
|
229
|
+
kind_or_agent_list = [:input, :output_with_router, @labels.values, :filter, :output].flatten
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
kind_or_agent_list.reverse! if desc
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
kind_or_agent_list.each do |kind|
|
|
236
|
+
if kind.respond_to?(:lifecycle)
|
|
237
|
+
agent = kind
|
|
238
|
+
agent.lifecycle(desc: desc) do |plugin, display_kind|
|
|
239
|
+
yield plugin, display_kind
|
|
240
|
+
end
|
|
241
|
+
else
|
|
242
|
+
list = if desc
|
|
243
|
+
lifecycle_control_list[kind].reverse
|
|
244
|
+
else
|
|
245
|
+
lifecycle_control_list[kind]
|
|
246
|
+
end
|
|
247
|
+
display_kind = (kind == :output_with_router ? :output : kind)
|
|
248
|
+
list.each do |instance|
|
|
249
|
+
if only_zero_downtime_restart_ready
|
|
250
|
+
next unless instance.respond_to?(:zero_downtime_restart_ready?) and instance.zero_downtime_restart_ready?
|
|
251
|
+
end
|
|
252
|
+
yield instance, display_kind
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
if kind_callback
|
|
256
|
+
kind_callback.call
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def start(kind_or_agent_list: nil)
|
|
262
|
+
lifecycle(desc: true, kind_or_agent_list: kind_or_agent_list) do |i| # instance
|
|
263
|
+
i.start unless i.started?
|
|
264
|
+
# Input#start sometimes emits lots of events with in_tail/`read_from_head true` case
|
|
265
|
+
# and it causes deadlock for small buffer/queue output. To avoid such problem,
|
|
266
|
+
# buffer related output threads should be run before `Input#start`.
|
|
267
|
+
# This is why after_start should be called immediately after start call.
|
|
268
|
+
# This depends on `desc: true` because calling plugin order of `desc: true` is
|
|
269
|
+
# Output, Filter, Label, Output with Router, then Input.
|
|
270
|
+
i.after_start unless i.after_started?
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
def flush!
|
|
275
|
+
log.info "flushing all buffer forcedly"
|
|
276
|
+
flushing_threads = []
|
|
277
|
+
lifecycle(desc: true) do |instance|
|
|
278
|
+
if instance.respond_to?(:force_flush)
|
|
279
|
+
t = Thread.new do
|
|
280
|
+
Thread.current.abort_on_exception = true
|
|
281
|
+
begin
|
|
282
|
+
instance.force_flush
|
|
283
|
+
rescue => e
|
|
284
|
+
log.warn "unexpected error while flushing buffer", plugin: instance.class, plugin_id: instance.plugin_id, error: e
|
|
285
|
+
log.warn_backtrace
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
flushing_threads << t
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
flushing_threads.each{|t| t.join }
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def cancel_source_only!
|
|
295
|
+
unless @source_only_mode.enabled?
|
|
296
|
+
log.info "do nothing for canceling with-source-only because the current mode is not with-source-only."
|
|
297
|
+
return
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
log.info "cancel with-source-only mode and start the other plugins"
|
|
301
|
+
all_plugins = [:input, :output_with_router, @labels.values, :filter, :output].flatten.reverse
|
|
302
|
+
start(kind_or_agent_list: all_plugins)
|
|
303
|
+
|
|
304
|
+
lifecycle_control_list[:input].each(&:event_emitter_cancel_source_only)
|
|
305
|
+
|
|
306
|
+
# Want to make sure that the source_only_router finishes all process before
|
|
307
|
+
# shutting down the agent.
|
|
308
|
+
# Strictly speaking, it would be necessary to have exclusive lock between
|
|
309
|
+
# EventRouter and the shutting down process of this agent.
|
|
310
|
+
# However, adding lock to EventRouter would worsen its performance, and
|
|
311
|
+
# the entire shutting down process does not care about it either.
|
|
312
|
+
# So, sleep here just in case.
|
|
313
|
+
sleep 1
|
|
314
|
+
|
|
315
|
+
shutdown(kind_or_agent_list: [@source_only_buffer_agent])
|
|
316
|
+
@source_only_buffer_agent = nil
|
|
317
|
+
|
|
318
|
+
# This agent can stop after flushing its all buffer, but it is not implemented for now.
|
|
319
|
+
log.info "starts the loading agent for with-source-only"
|
|
320
|
+
setup_source_only_buffer_agent(flush: true)
|
|
321
|
+
start(kind_or_agent_list: [@source_only_buffer_agent])
|
|
322
|
+
|
|
323
|
+
@source_only_mode.disable!
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
def shutdown(kind_or_agent_list: nil)
|
|
327
|
+
# Fluentd's shutdown sequence is stop, before_shutdown, shutdown, after_shutdown, close, terminate for plugins
|
|
328
|
+
# These method callers does `rescue Exception` to call methods of shutdown sequence as far as possible
|
|
329
|
+
# if plugin methods does something like infinite recursive call, `exit`, unregistering signal handlers or others.
|
|
330
|
+
# Plugins should be separated and be in sandbox to protect data in each plugins/buffers.
|
|
331
|
+
|
|
332
|
+
lifecycle_safe_sequence = ->(method, checker) {
|
|
333
|
+
lifecycle(kind_or_agent_list: kind_or_agent_list) do |instance, kind|
|
|
334
|
+
begin
|
|
335
|
+
log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
|
|
336
|
+
instance.__send__(method) unless instance.__send__(checker)
|
|
337
|
+
rescue Exception => e
|
|
338
|
+
log.warn "unexpected error while calling #{method} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
|
|
339
|
+
log.warn_backtrace
|
|
340
|
+
end
|
|
341
|
+
end
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
lifecycle_unsafe_sequence = ->(method, checker) {
|
|
345
|
+
operation = case method
|
|
346
|
+
when :shutdown then "shutting down"
|
|
347
|
+
when :close then "closing"
|
|
348
|
+
else
|
|
349
|
+
raise "BUG: unknown method name '#{method}'"
|
|
350
|
+
end
|
|
351
|
+
operation_threads = []
|
|
352
|
+
callback = ->(){
|
|
353
|
+
operation_threads.each{|t| t.join }
|
|
354
|
+
operation_threads.clear
|
|
355
|
+
}
|
|
356
|
+
lifecycle(kind_callback: callback, kind_or_agent_list: kind_or_agent_list) do |instance, kind|
|
|
357
|
+
t = Thread.new do
|
|
358
|
+
Thread.current.abort_on_exception = true
|
|
359
|
+
begin
|
|
360
|
+
if method == :shutdown
|
|
361
|
+
# To avoid Input#shutdown and Output#before_shutdown mismatch problem, combine before_shutdown and shutdown call in one sequence.
|
|
362
|
+
# The problem is in_tail flushes buffered multiline in shutdown but output's flush_at_shutdown is invoked in before_shutdown
|
|
363
|
+
operation = "preparing shutdown" # for logging
|
|
364
|
+
log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
|
|
365
|
+
begin
|
|
366
|
+
instance.__send__(:before_shutdown) unless instance.__send__(:before_shutdown?)
|
|
367
|
+
rescue Exception => e
|
|
368
|
+
log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
|
|
369
|
+
log.warn_backtrace
|
|
370
|
+
end
|
|
371
|
+
operation = "shutting down"
|
|
372
|
+
log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
|
|
373
|
+
instance.__send__(:shutdown) unless instance.__send__(:shutdown?)
|
|
374
|
+
else
|
|
375
|
+
log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
|
|
376
|
+
instance.__send__(method) unless instance.__send__(checker)
|
|
377
|
+
end
|
|
378
|
+
rescue Exception => e
|
|
379
|
+
log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
|
|
380
|
+
log.warn_backtrace
|
|
381
|
+
end
|
|
382
|
+
end
|
|
383
|
+
operation_threads << t
|
|
384
|
+
end
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
lifecycle_safe_sequence.call(:stop, :stopped?)
|
|
388
|
+
|
|
389
|
+
# before_shutdown does force_flush for output plugins: it should block, so it's unsafe operation
|
|
390
|
+
lifecycle_unsafe_sequence.call(:shutdown, :shutdown?)
|
|
391
|
+
|
|
392
|
+
lifecycle_safe_sequence.call(:after_shutdown, :after_shutdown?)
|
|
393
|
+
|
|
394
|
+
lifecycle_unsafe_sequence.call(:close, :closed?)
|
|
395
|
+
|
|
396
|
+
lifecycle_safe_sequence.call(:terminate, :terminated?)
|
|
397
|
+
|
|
398
|
+
cleanup_source_only_buffer_agent unless kind_or_agent_list
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
def suppress_interval(interval_time)
|
|
402
|
+
@suppress_emit_error_log_interval = interval_time
|
|
403
|
+
@next_emit_error_log_time = Time.now.to_i
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
def add_source(type, conf)
|
|
407
|
+
log_type = conf.for_this_worker? ? :default : :worker0
|
|
408
|
+
log.info log_type, "adding source", type: type
|
|
409
|
+
|
|
410
|
+
input = Plugin.new_input(type)
|
|
411
|
+
# <source> emits events to the top-level event router (RootAgent#event_router).
|
|
412
|
+
# Input#configure overwrites event_router to a label's event_router if it has `@label` parameter.
|
|
413
|
+
# See also 'fluentd/plugin/input.rb'
|
|
414
|
+
input.context_router = @event_router
|
|
415
|
+
input.configure(conf)
|
|
416
|
+
input.event_emitter_apply_source_only if @source_only_mode.enabled?
|
|
417
|
+
if @enable_input_metrics
|
|
418
|
+
input.event_emitter_router(conf['@label']).add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
|
|
419
|
+
end
|
|
420
|
+
@inputs << input
|
|
421
|
+
|
|
422
|
+
input
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
def add_label(name)
|
|
426
|
+
label = Label.new(name, log: log)
|
|
427
|
+
raise ConfigError, "Section <label #{name}> appears twice" if @labels[name]
|
|
428
|
+
label.root_agent = self
|
|
429
|
+
@labels[name] = label
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
def find_label(label_name)
|
|
433
|
+
if label = @labels[label_name]
|
|
434
|
+
label
|
|
435
|
+
else
|
|
436
|
+
raise ArgumentError, "#{label_name} label not found"
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
def emit_error_event(tag, time, record, error)
|
|
441
|
+
error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}
|
|
442
|
+
if @error_collector
|
|
443
|
+
# A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
|
|
444
|
+
log.warn "send an error event to @ERROR:", error_info
|
|
445
|
+
@error_collector.emit(tag, time, record)
|
|
446
|
+
else
|
|
447
|
+
error_info[:record] = record
|
|
448
|
+
log.warn "dump an error event:", error_info
|
|
449
|
+
end
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
def handle_emits_error(tag, es, error)
|
|
453
|
+
error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
|
|
454
|
+
if @error_collector
|
|
455
|
+
log.warn "send an error event stream to @ERROR:", error_info
|
|
456
|
+
@error_collector.emit_stream(tag, es)
|
|
457
|
+
else
|
|
458
|
+
now = Time.now.to_i
|
|
459
|
+
if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
|
|
460
|
+
log.warn "emit transaction failed:", error_info
|
|
461
|
+
log.warn_backtrace
|
|
462
|
+
@next_emit_error_log_time = now + @suppress_emit_error_log_interval
|
|
463
|
+
end
|
|
464
|
+
raise error
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
end
|
|
468
|
+
end
|