fluentd 0.14.4-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +26 -0
- data/.travis.yml +45 -0
- data/AUTHORS +2 -0
- data/CONTRIBUTING.md +35 -0
- data/COPYING +14 -0
- data/ChangeLog +276 -0
- data/Gemfile +9 -0
- data/README.md +51 -0
- data/Rakefile +53 -0
- data/Vagrantfile +17 -0
- data/appveyor.yml +41 -0
- data/bin/fluent-debug +5 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +11 -0
- data/example/in_http.conf +14 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_buffered_null.conf +32 -0
- data/example/out_copy.conf +20 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +51 -0
- data/lib/fluent/agent.rb +194 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/cat.rb +319 -0
- data/lib/fluent/command/debug.rb +102 -0
- data/lib/fluent/command/fluentd.rb +273 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +677 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +180 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config.rb +56 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +366 -0
- data/lib/fluent/config/dsl.rb +149 -0
- data/lib/fluent/config/element.rb +218 -0
- data/lib/fluent/config/error.rb +26 -0
- data/lib/fluent/config/literal_parser.rb +251 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +212 -0
- data/lib/fluent/config/types.rb +136 -0
- data/lib/fluent/config/v1_parser.rb +190 -0
- data/lib/fluent/configurable.rb +176 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +220 -0
- data/lib/fluent/env.rb +27 -0
- data/lib/fluent/event.rb +287 -0
- data/lib/fluent/event_router.rb +259 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +38 -0
- data/lib/fluent/load.rb +36 -0
- data/lib/fluent/log.rb +445 -0
- data/lib/fluent/match.rb +141 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +26 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin.rb +161 -0
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +130 -0
- data/lib/fluent/plugin/buf_file.rb +154 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer.rb +603 -0
- data/lib/fluent/plugin/buffer/chunk.rb +160 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +75 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +45 -0
- data/lib/fluent/plugin/formatter_csv.rb +47 -0
- data/lib/fluent/plugin/formatter_hash.rb +29 -0
- data/lib/fluent/plugin/formatter_json.rb +44 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
- data/lib/fluent/plugin/formatter_out_file.rb +78 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +74 -0
- data/lib/fluent/plugin/in_debug_agent.rb +64 -0
- data/lib/fluent/plugin/in_dummy.rb +135 -0
- data/lib/fluent/plugin/in_exec.rb +149 -0
- data/lib/fluent/plugin/in_forward.rb +366 -0
- data/lib/fluent/plugin/in_gc_stat.rb +52 -0
- data/lib/fluent/plugin/in_http.rb +422 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
- data/lib/fluent/plugin/in_object_space.rb +90 -0
- data/lib/fluent/plugin/in_syslog.rb +204 -0
- data/lib/fluent/plugin/in_tail.rb +838 -0
- data/lib/fluent/plugin/in_tcp.rb +41 -0
- data/lib/fluent/plugin/in_udp.rb +37 -0
- data/lib/fluent/plugin/in_unix.rb +201 -0
- data/lib/fluent/plugin/input.rb +33 -0
- data/lib/fluent/plugin/multi_output.rb +95 -0
- data/lib/fluent/plugin/out_buffered_null.rb +59 -0
- data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
- data/lib/fluent/plugin/out_copy.rb +42 -0
- data/lib/fluent/plugin/out_exec.rb +114 -0
- data/lib/fluent/plugin/out_exec_filter.rb +393 -0
- data/lib/fluent/plugin/out_file.rb +167 -0
- data/lib/fluent/plugin/out_forward.rb +646 -0
- data/lib/fluent/plugin/out_null.rb +27 -0
- data/lib/fluent/plugin/out_relabel.rb +28 -0
- data/lib/fluent/plugin/out_roundrobin.rb +80 -0
- data/lib/fluent/plugin/out_stdout.rb +48 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1020 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +175 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +33 -0
- data/lib/fluent/plugin/parser_json.rb +79 -0
- data/lib/fluent/plugin/parser_ltsv.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +104 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +73 -0
- data/lib/fluent/plugin/parser_syslog.rb +82 -0
- data/lib/fluent/plugin/parser_tsv.rb +37 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +132 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin_helper.rb +42 -0
- data/lib/fluent/plugin_helper/child_process.rb +298 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
- data/lib/fluent/plugin_helper/event_loop.rb +118 -0
- data/lib/fluent/plugin_helper/formatter.rb +149 -0
- data/lib/fluent/plugin_helper/inject.rb +125 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/retry_state.rb +177 -0
- data/lib/fluent/plugin_helper/storage.rb +331 -0
- data/lib/fluent/plugin_helper/thread.rb +147 -0
- data/lib/fluent/plugin_helper/timer.rb +90 -0
- data/lib/fluent/plugin_id.rb +63 -0
- data/lib/fluent/process.rb +504 -0
- data/lib/fluent/registry.rb +99 -0
- data/lib/fluent/root_agent.rb +314 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/supervisor.rb +680 -0
- data/lib/fluent/system_config.rb +122 -0
- data/lib/fluent/test.rb +56 -0
- data/lib/fluent/test/base.rb +85 -0
- data/lib/fluent/test/driver/base.rb +179 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +125 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +52 -0
- data/lib/fluent/test/driver/output.rb +76 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +79 -0
- data/lib/fluent/test/input_test.rb +172 -0
- data/lib/fluent/test/log.rb +73 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/time.rb +175 -0
- data/lib/fluent/timezone.rb +133 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +71 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +82 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +507 -0
- data/test/config/test_configurable.rb +1194 -0
- data/test/config/test_configure_proxy.rb +386 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +403 -0
- data/test/config/test_literal_parser.rb +297 -0
- data/test/config/test_section.rb +184 -0
- data/test/config/test_system_config.rb +120 -0
- data/test/config/test_types.rb +171 -0
- data/test/helper.rb +119 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +75 -0
- data/test/plugin/test_buf_file.rb +571 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1200 -0
- data/test/plugin/test_buffer_chunk.rb +168 -0
- data/test/plugin/test_buffer_file_chunk.rb +771 -0
- data/test/plugin/test_buffer_memory_chunk.rb +265 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_filter.rb +353 -0
- data/test/plugin/test_filter_grep.rb +119 -0
- data/test/plugin/test_filter_record_transformer.rb +600 -0
- data/test/plugin/test_filter_stdout.rb +211 -0
- data/test/plugin/test_formatter_csv.rb +94 -0
- data/test/plugin/test_formatter_json.rb +30 -0
- data/test/plugin/test_formatter_ltsv.rb +52 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_in_debug_agent.rb +28 -0
- data/test/plugin/test_in_dummy.rb +188 -0
- data/test/plugin/test_in_exec.rb +133 -0
- data/test/plugin/test_in_forward.rb +635 -0
- data/test/plugin/test_in_gc_stat.rb +39 -0
- data/test/plugin/test_in_http.rb +442 -0
- data/test/plugin/test_in_monitor_agent.rb +329 -0
- data/test/plugin/test_in_object_space.rb +64 -0
- data/test/plugin/test_in_syslog.rb +205 -0
- data/test/plugin/test_in_tail.rb +1001 -0
- data/test/plugin/test_in_tcp.rb +102 -0
- data/test/plugin/test_in_udp.rb +121 -0
- data/test/plugin/test_in_unix.rb +126 -0
- data/test/plugin/test_input.rb +122 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_buffered_null.rb +79 -0
- data/test/plugin/test_out_buffered_stdout.rb +122 -0
- data/test/plugin/test_out_copy.rb +160 -0
- data/test/plugin/test_out_exec.rb +155 -0
- data/test/plugin/test_out_exec_filter.rb +262 -0
- data/test/plugin/test_out_file.rb +383 -0
- data/test/plugin/test_out_forward.rb +590 -0
- data/test/plugin/test_out_null.rb +29 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_stdout.rb +92 -0
- data/test/plugin/test_out_stream.rb +93 -0
- data/test/plugin/test_output.rb +568 -0
- data/test/plugin/test_output_as_buffered.rb +1604 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +839 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +38 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_base.rb +32 -0
- data/test/plugin/test_parser_csv.rb +104 -0
- data/test/plugin/test_parser_json.rb +107 -0
- data/test/plugin/test_parser_labeled_tsv.rb +129 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +48 -0
- data/test/plugin/test_parser_none.rb +53 -0
- data/test/plugin/test_parser_regexp.rb +277 -0
- data/test/plugin/test_parser_syslog.rb +66 -0
- data/test/plugin/test_parser_time.rb +46 -0
- data/test/plugin/test_parser_tsv.rb +121 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +8 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/test_child_process.rb +608 -0
- data/test/plugin_helper/test_compat_parameters.rb +242 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_formatter.rb +252 -0
- data/test/plugin_helper/test_inject.rb +487 -0
- data/test/plugin_helper/test_parser.rb +263 -0
- data/test/plugin_helper/test_retry_state.rb +399 -0
- data/test/plugin_helper/test_storage.rb +521 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +131 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_config.rb +179 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_event.rb +329 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +184 -0
- data/test/test_filter.rb +121 -0
- data/test/test_formatter.rb +319 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +572 -0
- data/test/test_match.rb +137 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_output.rb +214 -0
- data/test/test_plugin_classes.rb +136 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_process.rb +48 -0
- data/test/test_root_agent.rb +278 -0
- data/test/test_supervisor.rb +339 -0
- data/test/test_time_formatter.rb +186 -0
- data/test/test_unique_id.rb +47 -0
- metadata +823 -0
@@ -0,0 +1,331 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'forwardable'
|
18
|
+
|
19
|
+
require 'fluent/plugin'
|
20
|
+
require 'fluent/plugin/storage'
|
21
|
+
require 'fluent/plugin_helper/timer'
|
22
|
+
require 'fluent/config/element'
|
23
|
+
require 'fluent/configurable'
|
24
|
+
|
25
|
+
module Fluent
|
26
|
+
module PluginHelper
|
27
|
+
module Storage
|
28
|
+
include Fluent::PluginHelper::Timer
|
29
|
+
|
30
|
+
StorageState = Struct.new(:storage, :running)
|
31
|
+
|
32
|
+
def storage_create(usage: '', type: nil, conf: nil, default_type: nil)
|
33
|
+
s = @_storages[usage]
|
34
|
+
if s && s.running
|
35
|
+
return s.storage
|
36
|
+
elsif s
|
37
|
+
# storage is already created, but not loaded / started
|
38
|
+
else # !s
|
39
|
+
type = if type
|
40
|
+
type
|
41
|
+
elsif conf && conf.respond_to?(:[])
|
42
|
+
raise Fluent::ConfigError, "@type is required in <storage>" unless conf['@type']
|
43
|
+
conf['@type']
|
44
|
+
elsif default_type
|
45
|
+
default_type
|
46
|
+
else
|
47
|
+
raise ArgumentError, "BUG: both type and conf are not specified"
|
48
|
+
end
|
49
|
+
storage = Plugin.new_storage(type, parent: self)
|
50
|
+
config = case conf
|
51
|
+
when Fluent::Config::Element
|
52
|
+
conf
|
53
|
+
when Hash
|
54
|
+
# in code, programmer may use symbols as keys, but Element needs strings
|
55
|
+
conf = Hash[conf.map{|k,v| [k.to_s, v]}]
|
56
|
+
Fluent::Config::Element.new('storage', usage, conf, [])
|
57
|
+
when nil
|
58
|
+
Fluent::Config::Element.new('storage', usage, {}, [])
|
59
|
+
else
|
60
|
+
raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
|
61
|
+
end
|
62
|
+
storage.configure(config)
|
63
|
+
if @_storages_started
|
64
|
+
storage.start
|
65
|
+
end
|
66
|
+
s = @_storages[usage] = StorageState.new(wrap_instance(storage), false)
|
67
|
+
end
|
68
|
+
|
69
|
+
s.storage
|
70
|
+
end
|
71
|
+
|
72
|
+
module StorageParams
|
73
|
+
include Fluent::Configurable
|
74
|
+
# minimum section definition to instantiate storage plugin instances
|
75
|
+
config_section :storage, required: false, multi: true, param_name: :storage_configs do
|
76
|
+
config_argument :usage, :string, default: ''
|
77
|
+
config_param :@type, :string, default: Fluent::Plugin::Storage::DEFAULT_TYPE
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def self.included(mod)
|
82
|
+
mod.include StorageParams
|
83
|
+
end
|
84
|
+
|
85
|
+
attr_reader :_storages # for tests
|
86
|
+
|
87
|
+
def initialize
|
88
|
+
super
|
89
|
+
@_storages_started = false
|
90
|
+
@_storages = {} # usage => storage_state
|
91
|
+
end
|
92
|
+
|
93
|
+
def configure(conf)
|
94
|
+
super
|
95
|
+
|
96
|
+
@storage_configs.each do |section|
|
97
|
+
if @_storages[section.usage]
|
98
|
+
raise Fluent::ConfigError, "duplicated storages configured: #{section.usage}"
|
99
|
+
end
|
100
|
+
storage = Plugin.new_storage(section[:@type], parent: self)
|
101
|
+
storage.configure(section.corresponding_config_element)
|
102
|
+
@_storages[section.usage] = StorageState.new(wrap_instance(storage), false)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def start
|
107
|
+
super
|
108
|
+
|
109
|
+
@_storages_started = true
|
110
|
+
@_storages.each_pair do |usage, s|
|
111
|
+
s.storage.start
|
112
|
+
s.storage.load
|
113
|
+
|
114
|
+
if s.storage.autosave && !s.storage.persistent
|
115
|
+
timer_execute(:storage_autosave, s.storage.autosave_interval, repeat: true) do
|
116
|
+
begin
|
117
|
+
s.storage.save
|
118
|
+
rescue => e
|
119
|
+
log.error "plugin storage failed to save its data", usage: usage, type: type, error: e
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
s.running = true
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def storage_operate(method_name, &block)
|
128
|
+
@_storages.each_pair do |usage, s|
|
129
|
+
begin
|
130
|
+
block.call(s) if block_given?
|
131
|
+
s.storage.send(method_name)
|
132
|
+
rescue => e
|
133
|
+
log.error "unexpected error while #{method_name}", usage: usage, storage: s.storage, error: e
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def stop
|
139
|
+
super
|
140
|
+
# timer stops automatically in super
|
141
|
+
storage_operate(:stop)
|
142
|
+
end
|
143
|
+
|
144
|
+
def before_shutdown
|
145
|
+
storage_operate(:before_shutdown)
|
146
|
+
super
|
147
|
+
end
|
148
|
+
|
149
|
+
def shutdown
|
150
|
+
storage_operate(:shutdown) do |s|
|
151
|
+
s.storage.save if s.storage.save_at_shutdown
|
152
|
+
end
|
153
|
+
super
|
154
|
+
end
|
155
|
+
|
156
|
+
def after_shutdown
|
157
|
+
storage_operate(:after_shutdown)
|
158
|
+
super
|
159
|
+
end
|
160
|
+
|
161
|
+
def close
|
162
|
+
storage_operate(:close){|s| s.running = false }
|
163
|
+
super
|
164
|
+
end
|
165
|
+
|
166
|
+
def terminate
|
167
|
+
storage_operate(:terminate)
|
168
|
+
@_storages = {}
|
169
|
+
super
|
170
|
+
end
|
171
|
+
|
172
|
+
def wrap_instance(storage)
|
173
|
+
if storage.persistent && storage.persistent_always?
|
174
|
+
storage
|
175
|
+
elsif storage.persistent
|
176
|
+
PersistentWrapper.new(storage)
|
177
|
+
elsif !storage.synchronized?
|
178
|
+
SynchronizeWrapper.new(storage)
|
179
|
+
else
|
180
|
+
storage
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
class PersistentWrapper
|
185
|
+
# PersistentWrapper always provides synchronized operations
|
186
|
+
extend Forwardable
|
187
|
+
|
188
|
+
def initialize(storage)
|
189
|
+
@storage = storage
|
190
|
+
@monitor = Monitor.new
|
191
|
+
end
|
192
|
+
|
193
|
+
def_delegators :@storage, :autosave_interval, :save_at_shutdown
|
194
|
+
def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
|
195
|
+
def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
|
196
|
+
|
197
|
+
def persistent_always?
|
198
|
+
true
|
199
|
+
end
|
200
|
+
|
201
|
+
def persistent
|
202
|
+
true
|
203
|
+
end
|
204
|
+
|
205
|
+
def autosave
|
206
|
+
false
|
207
|
+
end
|
208
|
+
|
209
|
+
def synchronized?
|
210
|
+
true
|
211
|
+
end
|
212
|
+
|
213
|
+
def implementation
|
214
|
+
@storage
|
215
|
+
end
|
216
|
+
|
217
|
+
def load
|
218
|
+
@monitor.synchronize do
|
219
|
+
@storage.load
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
def save
|
224
|
+
@monitor.synchronize do
|
225
|
+
@storage.save
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def get(key)
|
230
|
+
@monitor.synchronize do
|
231
|
+
@storage.load
|
232
|
+
@storage.get(key)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def fetch(key, defval)
|
237
|
+
@monitor.synchronize do
|
238
|
+
@storage.load
|
239
|
+
@storage.fetch(key, defval)
|
240
|
+
end
|
241
|
+
end
|
242
|
+
|
243
|
+
def put(key, value)
|
244
|
+
@monitor.synchronize do
|
245
|
+
@storage.load
|
246
|
+
@storage.put(key, value)
|
247
|
+
@storage.save
|
248
|
+
value
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
def delete(key)
|
253
|
+
@monitor.synchronize do
|
254
|
+
@storage.load
|
255
|
+
val = @storage.delete(key)
|
256
|
+
@storage.save
|
257
|
+
val
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
def update(key, &block)
|
262
|
+
@monitor.synchronize do
|
263
|
+
@storage.load
|
264
|
+
v = block.call(@storage.get(key))
|
265
|
+
@storage.put(key, v)
|
266
|
+
@storage.save
|
267
|
+
v
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
class SynchronizeWrapper
|
273
|
+
extend Forwardable
|
274
|
+
|
275
|
+
def initialize(storage)
|
276
|
+
@storage = storage
|
277
|
+
@mutex = Mutex.new
|
278
|
+
end
|
279
|
+
|
280
|
+
def_delegators :@storage, :persistent, :autosave, :autosave_interval, :save_at_shutdown
|
281
|
+
def_delegators :@storage, :persistent_always?
|
282
|
+
def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
|
283
|
+
def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
|
284
|
+
|
285
|
+
def synchronized?
|
286
|
+
true
|
287
|
+
end
|
288
|
+
|
289
|
+
def implementation
|
290
|
+
@storage
|
291
|
+
end
|
292
|
+
|
293
|
+
def load
|
294
|
+
@mutex.synchronize do
|
295
|
+
@storage.load
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
def save
|
300
|
+
@mutex.synchronize do
|
301
|
+
@storage.save
|
302
|
+
end
|
303
|
+
end
|
304
|
+
|
305
|
+
def get(key)
|
306
|
+
@mutex.synchronize{ @storage.get(key) }
|
307
|
+
end
|
308
|
+
|
309
|
+
def fetch(key, defval)
|
310
|
+
@mutex.synchronize{ @storage.fetch(key, defval) }
|
311
|
+
end
|
312
|
+
|
313
|
+
def put(key, value)
|
314
|
+
@mutex.synchronize{ @storage.put(key, value) }
|
315
|
+
end
|
316
|
+
|
317
|
+
def delete(key)
|
318
|
+
@mutex.synchronize{ @storage.delete(key) }
|
319
|
+
end
|
320
|
+
|
321
|
+
def update(key, &block)
|
322
|
+
@mutex.synchronize do
|
323
|
+
v = block.call(@storage.get(key))
|
324
|
+
@storage.put(key, v)
|
325
|
+
v
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
end
|
330
|
+
end
|
331
|
+
end
|
@@ -0,0 +1,147 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module PluginHelper
|
19
|
+
module Thread
|
20
|
+
THREAD_DEFAULT_WAIT_SECONDS = 1
|
21
|
+
|
22
|
+
# stop : mark callback thread as stopped
|
23
|
+
# shutdown : [-]
|
24
|
+
# close : correct stopped threads
|
25
|
+
# terminate: kill all threads
|
26
|
+
|
27
|
+
attr_reader :_threads # for test driver
|
28
|
+
|
29
|
+
def thread_current_running?
|
30
|
+
# checker for code in callback of thread_create
|
31
|
+
::Thread.current[:_fluentd_plugin_helper_thread_running] || false
|
32
|
+
end
|
33
|
+
|
34
|
+
def thread_wait_until_start
|
35
|
+
until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && t[:_fluentd_plugin_helper_thread_started] } }
|
36
|
+
sleep 0.1
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def thread_wait_until_stop
|
41
|
+
until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && ![:_fluentd_plugin_helper_thread_running] } }
|
42
|
+
sleep 0.1
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# Ruby 2.2.3 or earlier (and all 2.1.x) cause bug about Threading ("Stack consistency error")
|
47
|
+
# by passing splatted argument to `yield`
|
48
|
+
# https://bugs.ruby-lang.org/issues/11027
|
49
|
+
# We can enable to pass arguments after expire of Ruby 2.1 (& older 2.2.x)
|
50
|
+
# def thread_create(title, *args)
|
51
|
+
# Thread.new(*args) do |*t_args|
|
52
|
+
# yield *t_args
|
53
|
+
def thread_create(title)
|
54
|
+
raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
|
55
|
+
raise ArgumentError, "BUG: callback not specified" unless block_given?
|
56
|
+
m = Mutex.new
|
57
|
+
m.lock
|
58
|
+
thread = ::Thread.new do
|
59
|
+
m.lock # run thread after that thread is successfully set into @_threads
|
60
|
+
m.unlock
|
61
|
+
thread_exit = false
|
62
|
+
::Thread.current[:_fluentd_plugin_helper_thread_title] = title
|
63
|
+
::Thread.current[:_fluentd_plugin_helper_thread_started] = true
|
64
|
+
::Thread.current[:_fluentd_plugin_helper_thread_running] = true
|
65
|
+
begin
|
66
|
+
yield
|
67
|
+
thread_exit = true
|
68
|
+
rescue Exception => e
|
69
|
+
log.warn "thread exited by unexpected error", plugin: self.class, title: title, error: e
|
70
|
+
thread_exit = true
|
71
|
+
raise
|
72
|
+
ensure
|
73
|
+
unless thread_exit
|
74
|
+
log.warn "thread doesn't exit correctly (killed or other reason)", plugin: self.class, title: title, thread: ::Thread.current, error: $!
|
75
|
+
end
|
76
|
+
@_threads_mutex.synchronize do
|
77
|
+
@_threads.delete(::Thread.current.object_id)
|
78
|
+
end
|
79
|
+
::Thread.current[:_fluentd_plugin_helper_thread_running] = false
|
80
|
+
end
|
81
|
+
end
|
82
|
+
thread.abort_on_exception = true
|
83
|
+
@_threads_mutex.synchronize do
|
84
|
+
@_threads[thread.object_id] = thread
|
85
|
+
end
|
86
|
+
m.unlock
|
87
|
+
thread
|
88
|
+
end
|
89
|
+
|
90
|
+
def thread_exist?(title)
|
91
|
+
@_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
|
92
|
+
end
|
93
|
+
|
94
|
+
def thread_started?(title)
|
95
|
+
t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
|
96
|
+
t && t[:_fluentd_plugin_helper_thread_started]
|
97
|
+
end
|
98
|
+
|
99
|
+
def thread_running?(title)
|
100
|
+
t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
|
101
|
+
t && t[:_fluentd_plugin_helper_thread_running]
|
102
|
+
end
|
103
|
+
|
104
|
+
def initialize
|
105
|
+
super
|
106
|
+
@_threads_mutex = Mutex.new
|
107
|
+
@_threads = {}
|
108
|
+
@_thread_wait_seconds = THREAD_DEFAULT_WAIT_SECONDS
|
109
|
+
end
|
110
|
+
|
111
|
+
def stop
|
112
|
+
super
|
113
|
+
@_threads_mutex.synchronize do
|
114
|
+
@_threads.each_pair do |obj_id, thread|
|
115
|
+
thread[:_fluentd_plugin_helper_thread_running] = false
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def close
|
121
|
+
@_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
|
122
|
+
thread = @_threads[obj_id]
|
123
|
+
if !thread || thread.join(@_thread_wait_seconds)
|
124
|
+
@_threads_mutex.synchronize{ @_threads.delete(obj_id) }
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
super
|
129
|
+
end
|
130
|
+
|
131
|
+
def terminate
|
132
|
+
super
|
133
|
+
@_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
|
134
|
+
thread = @_threads[obj_id]
|
135
|
+
log.warn "killing existing thead", thread: thread
|
136
|
+
thread.kill if thread
|
137
|
+
end
|
138
|
+
@_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
|
139
|
+
thread = @_threads[obj_id]
|
140
|
+
thread.join
|
141
|
+
@_threads_mutex.synchronize{ @_threads.delete(obj_id) }
|
142
|
+
end
|
143
|
+
@_thread_wait_seconds = nil
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|