fluentd 0.14.4-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +26 -0
- data/.travis.yml +45 -0
- data/AUTHORS +2 -0
- data/CONTRIBUTING.md +35 -0
- data/COPYING +14 -0
- data/ChangeLog +276 -0
- data/Gemfile +9 -0
- data/README.md +51 -0
- data/Rakefile +53 -0
- data/Vagrantfile +17 -0
- data/appveyor.yml +41 -0
- data/bin/fluent-debug +5 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +11 -0
- data/example/in_http.conf +14 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_buffered_null.conf +32 -0
- data/example/out_copy.conf +20 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +51 -0
- data/lib/fluent/agent.rb +194 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/cat.rb +319 -0
- data/lib/fluent/command/debug.rb +102 -0
- data/lib/fluent/command/fluentd.rb +273 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +677 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +180 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config.rb +56 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +366 -0
- data/lib/fluent/config/dsl.rb +149 -0
- data/lib/fluent/config/element.rb +218 -0
- data/lib/fluent/config/error.rb +26 -0
- data/lib/fluent/config/literal_parser.rb +251 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +212 -0
- data/lib/fluent/config/types.rb +136 -0
- data/lib/fluent/config/v1_parser.rb +190 -0
- data/lib/fluent/configurable.rb +176 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +220 -0
- data/lib/fluent/env.rb +27 -0
- data/lib/fluent/event.rb +287 -0
- data/lib/fluent/event_router.rb +259 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +38 -0
- data/lib/fluent/load.rb +36 -0
- data/lib/fluent/log.rb +445 -0
- data/lib/fluent/match.rb +141 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +26 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin.rb +161 -0
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +130 -0
- data/lib/fluent/plugin/buf_file.rb +154 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer.rb +603 -0
- data/lib/fluent/plugin/buffer/chunk.rb +160 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +75 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +45 -0
- data/lib/fluent/plugin/formatter_csv.rb +47 -0
- data/lib/fluent/plugin/formatter_hash.rb +29 -0
- data/lib/fluent/plugin/formatter_json.rb +44 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
- data/lib/fluent/plugin/formatter_out_file.rb +78 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +74 -0
- data/lib/fluent/plugin/in_debug_agent.rb +64 -0
- data/lib/fluent/plugin/in_dummy.rb +135 -0
- data/lib/fluent/plugin/in_exec.rb +149 -0
- data/lib/fluent/plugin/in_forward.rb +366 -0
- data/lib/fluent/plugin/in_gc_stat.rb +52 -0
- data/lib/fluent/plugin/in_http.rb +422 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
- data/lib/fluent/plugin/in_object_space.rb +90 -0
- data/lib/fluent/plugin/in_syslog.rb +204 -0
- data/lib/fluent/plugin/in_tail.rb +838 -0
- data/lib/fluent/plugin/in_tcp.rb +41 -0
- data/lib/fluent/plugin/in_udp.rb +37 -0
- data/lib/fluent/plugin/in_unix.rb +201 -0
- data/lib/fluent/plugin/input.rb +33 -0
- data/lib/fluent/plugin/multi_output.rb +95 -0
- data/lib/fluent/plugin/out_buffered_null.rb +59 -0
- data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
- data/lib/fluent/plugin/out_copy.rb +42 -0
- data/lib/fluent/plugin/out_exec.rb +114 -0
- data/lib/fluent/plugin/out_exec_filter.rb +393 -0
- data/lib/fluent/plugin/out_file.rb +167 -0
- data/lib/fluent/plugin/out_forward.rb +646 -0
- data/lib/fluent/plugin/out_null.rb +27 -0
- data/lib/fluent/plugin/out_relabel.rb +28 -0
- data/lib/fluent/plugin/out_roundrobin.rb +80 -0
- data/lib/fluent/plugin/out_stdout.rb +48 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1020 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +175 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +33 -0
- data/lib/fluent/plugin/parser_json.rb +79 -0
- data/lib/fluent/plugin/parser_ltsv.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +104 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +73 -0
- data/lib/fluent/plugin/parser_syslog.rb +82 -0
- data/lib/fluent/plugin/parser_tsv.rb +37 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +132 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin_helper.rb +42 -0
- data/lib/fluent/plugin_helper/child_process.rb +298 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
- data/lib/fluent/plugin_helper/event_loop.rb +118 -0
- data/lib/fluent/plugin_helper/formatter.rb +149 -0
- data/lib/fluent/plugin_helper/inject.rb +125 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/retry_state.rb +177 -0
- data/lib/fluent/plugin_helper/storage.rb +331 -0
- data/lib/fluent/plugin_helper/thread.rb +147 -0
- data/lib/fluent/plugin_helper/timer.rb +90 -0
- data/lib/fluent/plugin_id.rb +63 -0
- data/lib/fluent/process.rb +504 -0
- data/lib/fluent/registry.rb +99 -0
- data/lib/fluent/root_agent.rb +314 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/supervisor.rb +680 -0
- data/lib/fluent/system_config.rb +122 -0
- data/lib/fluent/test.rb +56 -0
- data/lib/fluent/test/base.rb +85 -0
- data/lib/fluent/test/driver/base.rb +179 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +125 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +52 -0
- data/lib/fluent/test/driver/output.rb +76 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +79 -0
- data/lib/fluent/test/input_test.rb +172 -0
- data/lib/fluent/test/log.rb +73 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/time.rb +175 -0
- data/lib/fluent/timezone.rb +133 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +71 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +82 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +507 -0
- data/test/config/test_configurable.rb +1194 -0
- data/test/config/test_configure_proxy.rb +386 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +403 -0
- data/test/config/test_literal_parser.rb +297 -0
- data/test/config/test_section.rb +184 -0
- data/test/config/test_system_config.rb +120 -0
- data/test/config/test_types.rb +171 -0
- data/test/helper.rb +119 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +75 -0
- data/test/plugin/test_buf_file.rb +571 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1200 -0
- data/test/plugin/test_buffer_chunk.rb +168 -0
- data/test/plugin/test_buffer_file_chunk.rb +771 -0
- data/test/plugin/test_buffer_memory_chunk.rb +265 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_filter.rb +353 -0
- data/test/plugin/test_filter_grep.rb +119 -0
- data/test/plugin/test_filter_record_transformer.rb +600 -0
- data/test/plugin/test_filter_stdout.rb +211 -0
- data/test/plugin/test_formatter_csv.rb +94 -0
- data/test/plugin/test_formatter_json.rb +30 -0
- data/test/plugin/test_formatter_ltsv.rb +52 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_in_debug_agent.rb +28 -0
- data/test/plugin/test_in_dummy.rb +188 -0
- data/test/plugin/test_in_exec.rb +133 -0
- data/test/plugin/test_in_forward.rb +635 -0
- data/test/plugin/test_in_gc_stat.rb +39 -0
- data/test/plugin/test_in_http.rb +442 -0
- data/test/plugin/test_in_monitor_agent.rb +329 -0
- data/test/plugin/test_in_object_space.rb +64 -0
- data/test/plugin/test_in_syslog.rb +205 -0
- data/test/plugin/test_in_tail.rb +1001 -0
- data/test/plugin/test_in_tcp.rb +102 -0
- data/test/plugin/test_in_udp.rb +121 -0
- data/test/plugin/test_in_unix.rb +126 -0
- data/test/plugin/test_input.rb +122 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_buffered_null.rb +79 -0
- data/test/plugin/test_out_buffered_stdout.rb +122 -0
- data/test/plugin/test_out_copy.rb +160 -0
- data/test/plugin/test_out_exec.rb +155 -0
- data/test/plugin/test_out_exec_filter.rb +262 -0
- data/test/plugin/test_out_file.rb +383 -0
- data/test/plugin/test_out_forward.rb +590 -0
- data/test/plugin/test_out_null.rb +29 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_stdout.rb +92 -0
- data/test/plugin/test_out_stream.rb +93 -0
- data/test/plugin/test_output.rb +568 -0
- data/test/plugin/test_output_as_buffered.rb +1604 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +839 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +38 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_base.rb +32 -0
- data/test/plugin/test_parser_csv.rb +104 -0
- data/test/plugin/test_parser_json.rb +107 -0
- data/test/plugin/test_parser_labeled_tsv.rb +129 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +48 -0
- data/test/plugin/test_parser_none.rb +53 -0
- data/test/plugin/test_parser_regexp.rb +277 -0
- data/test/plugin/test_parser_syslog.rb +66 -0
- data/test/plugin/test_parser_time.rb +46 -0
- data/test/plugin/test_parser_tsv.rb +121 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +8 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/test_child_process.rb +608 -0
- data/test/plugin_helper/test_compat_parameters.rb +242 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_formatter.rb +252 -0
- data/test/plugin_helper/test_inject.rb +487 -0
- data/test/plugin_helper/test_parser.rb +263 -0
- data/test/plugin_helper/test_retry_state.rb +399 -0
- data/test/plugin_helper/test_storage.rb +521 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +131 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_config.rb +179 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_event.rb +329 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +184 -0
- data/test/test_filter.rb +121 -0
- data/test/test_formatter.rb +319 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +572 -0
- data/test/test_match.rb +137 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_output.rb +214 -0
- data/test/test_plugin_classes.rb +136 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_process.rb +48 -0
- data/test/test_root_agent.rb +278 -0
- data/test/test_supervisor.rb +339 -0
- data/test/test_time_formatter.rb +186 -0
- data/test/test_unique_id.rb +47 -0
- metadata +823 -0
@@ -0,0 +1,190 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'strscan'
|
18
|
+
require 'uri'
|
19
|
+
|
20
|
+
require 'fluent/config/error'
|
21
|
+
require 'fluent/config/basic_parser'
|
22
|
+
require 'fluent/config/literal_parser'
|
23
|
+
require 'fluent/config/element'
|
24
|
+
|
25
|
+
module Fluent
|
26
|
+
module Config
|
27
|
+
class V1Parser < LiteralParser
|
28
|
+
ELEMENT_NAME = /[a-zA-Z0-9_]+/
|
29
|
+
|
30
|
+
def self.parse(data, fname, basepath = Dir.pwd, eval_context = nil)
|
31
|
+
ss = StringScanner.new(data)
|
32
|
+
ps = V1Parser.new(ss, basepath, fname, eval_context)
|
33
|
+
ps.parse!
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(strscan, include_basepath, fname, eval_context)
|
37
|
+
super(strscan, eval_context)
|
38
|
+
@include_basepath = include_basepath
|
39
|
+
@fname = fname
|
40
|
+
end
|
41
|
+
|
42
|
+
def parse!
|
43
|
+
attrs, elems = parse_element(true, nil)
|
44
|
+
root = Element.new('ROOT', '', attrs, elems)
|
45
|
+
root.v1_config = true
|
46
|
+
|
47
|
+
spacing
|
48
|
+
unless eof?
|
49
|
+
parse_error! "expected EOF"
|
50
|
+
end
|
51
|
+
|
52
|
+
return root
|
53
|
+
end
|
54
|
+
|
55
|
+
ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
|
56
|
+
RESERVED_PARAMS = %W(@type @id @label @log_level)
|
57
|
+
|
58
|
+
def parse_element(root_element, elem_name, attrs = {}, elems = [])
|
59
|
+
while true
|
60
|
+
spacing
|
61
|
+
if eof?
|
62
|
+
if root_element
|
63
|
+
break
|
64
|
+
end
|
65
|
+
parse_error! "expected end tag '</#{elem_name}>' but got end of file"
|
66
|
+
end
|
67
|
+
|
68
|
+
if skip(/\<\//)
|
69
|
+
e_name = scan(ELEMENT_NAME)
|
70
|
+
spacing
|
71
|
+
unless skip(/\>/)
|
72
|
+
parse_error! "expected character in tag name"
|
73
|
+
end
|
74
|
+
unless line_end
|
75
|
+
parse_error! "expected end of line after end tag"
|
76
|
+
end
|
77
|
+
if e_name != elem_name
|
78
|
+
parse_error! "unmatched end tag"
|
79
|
+
end
|
80
|
+
break
|
81
|
+
|
82
|
+
elsif skip(/\</)
|
83
|
+
e_name = scan(ELEMENT_NAME)
|
84
|
+
spacing
|
85
|
+
e_arg = scan_nonquoted_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
|
86
|
+
spacing
|
87
|
+
unless skip(/\>/)
|
88
|
+
parse_error! "expected '>'"
|
89
|
+
end
|
90
|
+
unless line_end
|
91
|
+
parse_error! "expected end of line after tag"
|
92
|
+
end
|
93
|
+
e_arg ||= ''
|
94
|
+
# call parse_element recursively
|
95
|
+
e_attrs, e_elems = parse_element(false, e_name)
|
96
|
+
new_e = Element.new(e_name, e_arg, e_attrs, e_elems)
|
97
|
+
new_e.v1_config = true
|
98
|
+
elems << new_e
|
99
|
+
|
100
|
+
elsif root_element && skip(/(\@include|include)#{SPACING}/)
|
101
|
+
if !prev_match.start_with?('@')
|
102
|
+
$log.warn "'include' is deprecated. Use '@include' instead"
|
103
|
+
end
|
104
|
+
parse_include(attrs, elems)
|
105
|
+
|
106
|
+
else
|
107
|
+
k = scan_string(SPACING)
|
108
|
+
spacing_without_comment
|
109
|
+
if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
|
110
|
+
attrs[k] = ""
|
111
|
+
else
|
112
|
+
if k == '@include'
|
113
|
+
parse_include(attrs, elems)
|
114
|
+
elsif RESERVED_PARAMS.include?(k)
|
115
|
+
v = parse_literal
|
116
|
+
unless line_end
|
117
|
+
parse_error! "expected end of line"
|
118
|
+
end
|
119
|
+
attrs[k] = v
|
120
|
+
else
|
121
|
+
if k.start_with?('@')
|
122
|
+
if root_element || ELEM_SYMBOLS.include?(elem_name)
|
123
|
+
parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
|
124
|
+
else
|
125
|
+
# TODO: This is for backward compatibility. It will throw an error in the future.
|
126
|
+
$log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
v = parse_literal
|
131
|
+
unless line_end
|
132
|
+
parse_error! "expected end of line"
|
133
|
+
end
|
134
|
+
attrs[k] = v
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
return attrs, elems
|
141
|
+
end
|
142
|
+
|
143
|
+
def parse_include(attrs, elems)
|
144
|
+
uri = scan_string(LINE_END)
|
145
|
+
eval_include(attrs, elems, uri)
|
146
|
+
line_end
|
147
|
+
end
|
148
|
+
|
149
|
+
def eval_include(attrs, elems, uri)
|
150
|
+
u = URI.parse(uri)
|
151
|
+
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri # file path
|
152
|
+
# When the Windows absolute path then u.scheme.length == 1
|
153
|
+
# e.g. C:
|
154
|
+
path = u.path
|
155
|
+
if path[0] != ?/
|
156
|
+
pattern = File.expand_path("#{@include_basepath}/#{path}")
|
157
|
+
else
|
158
|
+
pattern = path
|
159
|
+
end
|
160
|
+
Dir.glob(pattern).sort.each { |entry|
|
161
|
+
basepath = File.dirname(entry)
|
162
|
+
fname = File.basename(entry)
|
163
|
+
data = File.read(entry)
|
164
|
+
data.force_encoding('UTF-8')
|
165
|
+
ss = StringScanner.new(data)
|
166
|
+
V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
|
167
|
+
}
|
168
|
+
else
|
169
|
+
require 'open-uri'
|
170
|
+
basepath = '/'
|
171
|
+
fname = path
|
172
|
+
data = open(uri) { |f| f.read }
|
173
|
+
data.force_encoding('UTF-8')
|
174
|
+
ss = StringScanner.new(data)
|
175
|
+
V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
|
176
|
+
end
|
177
|
+
|
178
|
+
rescue SystemCallError => e
|
179
|
+
cpe = ConfigParseError.new("include error #{uri}")
|
180
|
+
cpe.set_backtrace(e.backtrace)
|
181
|
+
raise cpe
|
182
|
+
end
|
183
|
+
|
184
|
+
# override
|
185
|
+
def error_sample
|
186
|
+
"#{@fname} #{super}"
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
@@ -0,0 +1,176 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/config/configure_proxy'
|
18
|
+
require 'fluent/config/section'
|
19
|
+
require 'fluent/config/error'
|
20
|
+
require 'fluent/registry'
|
21
|
+
require 'fluent/plugin'
|
22
|
+
require 'fluent/config/types'
|
23
|
+
|
24
|
+
module Fluent
|
25
|
+
module Configurable
|
26
|
+
def self.included(mod)
|
27
|
+
mod.extend(ClassMethods)
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize
|
31
|
+
super
|
32
|
+
# to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
|
33
|
+
proxy = self.class.merged_configure_proxy
|
34
|
+
proxy.params.keys.each do |name|
|
35
|
+
next if name.to_s.start_with?('@')
|
36
|
+
if proxy.defaults.has_key?(name)
|
37
|
+
instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
|
38
|
+
end
|
39
|
+
end
|
40
|
+
proxy.sections.keys.each do |name|
|
41
|
+
next if name.to_s.start_with?('@')
|
42
|
+
subproxy = proxy.sections[name]
|
43
|
+
if subproxy.multi?
|
44
|
+
instance_variable_set("@#{subproxy.variable_name}".to_sym, [])
|
45
|
+
else
|
46
|
+
instance_variable_set("@#{subproxy.variable_name}".to_sym, nil)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def configure(conf)
|
52
|
+
@config = conf
|
53
|
+
|
54
|
+
logger = self.respond_to?(:log) ? log : (defined?($log) ? $log : nil)
|
55
|
+
proxy = self.class.merged_configure_proxy
|
56
|
+
conf.corresponding_proxies << proxy
|
57
|
+
|
58
|
+
if self.respond_to?(:owner) && self.owner
|
59
|
+
owner_proxy = owner.class.merged_configure_proxy
|
60
|
+
if proxy.configured_in_section
|
61
|
+
owner_proxy = owner_proxy.sections[proxy.configured_in_section]
|
62
|
+
end
|
63
|
+
proxy.overwrite_defaults(owner_proxy) if owner_proxy
|
64
|
+
end
|
65
|
+
|
66
|
+
# In the nested section, can't get plugin class through proxies so get plugin class here
|
67
|
+
plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
|
68
|
+
root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
|
69
|
+
@config_root_section = root
|
70
|
+
|
71
|
+
root.instance_eval{ @params.keys }.each do |param_name|
|
72
|
+
next if param_name.to_s.start_with?('@')
|
73
|
+
varname = "@#{param_name}".to_sym
|
74
|
+
if (! root[param_name].nil?) || (instance_variable_defined?(varname) && instance_variable_get(varname).nil?)
|
75
|
+
instance_variable_set(varname, root[param_name])
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
self
|
80
|
+
end
|
81
|
+
|
82
|
+
def config
|
83
|
+
@masked_config ||= @config.to_masked_element
|
84
|
+
end
|
85
|
+
|
86
|
+
CONFIG_TYPE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/type_')
|
87
|
+
|
88
|
+
def self.register_type(type, callable = nil, &block)
|
89
|
+
callable ||= block
|
90
|
+
CONFIG_TYPE_REGISTRY.register(type, callable)
|
91
|
+
end
|
92
|
+
|
93
|
+
def self.lookup_type(type)
|
94
|
+
CONFIG_TYPE_REGISTRY.lookup(type)
|
95
|
+
end
|
96
|
+
|
97
|
+
{
|
98
|
+
string: Config::STRING_TYPE,
|
99
|
+
enum: Config::ENUM_TYPE,
|
100
|
+
integer: Config::INTEGER_TYPE,
|
101
|
+
float: Config::FLOAT_TYPE,
|
102
|
+
size: Config::SIZE_TYPE,
|
103
|
+
bool: Config::BOOL_TYPE,
|
104
|
+
time: Config::TIME_TYPE,
|
105
|
+
hash: Config::HASH_TYPE,
|
106
|
+
array: Config::ARRAY_TYPE,
|
107
|
+
}.each do |name, type|
|
108
|
+
register_type(name, type)
|
109
|
+
end
|
110
|
+
|
111
|
+
module ClassMethods
|
112
|
+
def configure_proxy_map
|
113
|
+
map = {}
|
114
|
+
self.define_singleton_method(:configure_proxy_map){ map }
|
115
|
+
map
|
116
|
+
end
|
117
|
+
|
118
|
+
def configure_proxy(mod_name)
|
119
|
+
map = configure_proxy_map
|
120
|
+
unless map[mod_name]
|
121
|
+
type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }
|
122
|
+
proxy = Fluent::Config::ConfigureProxy.new(mod_name, root: true, required: true, multi: false, type_lookup: type_lookup)
|
123
|
+
map[mod_name] = proxy
|
124
|
+
end
|
125
|
+
map[mod_name]
|
126
|
+
end
|
127
|
+
|
128
|
+
def configured_in(section_name)
|
129
|
+
configure_proxy(self.name).configured_in(section_name)
|
130
|
+
end
|
131
|
+
|
132
|
+
def config_param(name, type = nil, **kwargs, &block)
|
133
|
+
configure_proxy(self.name).config_param(name, type, **kwargs, &block)
|
134
|
+
# reserved names '@foo' are invalid as attr_accessor name
|
135
|
+
attr_accessor(name) unless kwargs[:skip_accessor] || Fluent::Config::Element::RESERVED_PARAMETERS.include?(name.to_s)
|
136
|
+
end
|
137
|
+
|
138
|
+
def config_set_default(name, defval)
|
139
|
+
configure_proxy(self.name).config_set_default(name, defval)
|
140
|
+
end
|
141
|
+
|
142
|
+
def config_set_desc(name, desc)
|
143
|
+
configure_proxy(self.name).config_set_desc(name, desc)
|
144
|
+
end
|
145
|
+
|
146
|
+
def config_section(name, **kwargs, &block)
|
147
|
+
section_already_exists = !!merged_configure_proxy.sections[name]
|
148
|
+
configure_proxy(self.name).config_section(name, **kwargs, &block)
|
149
|
+
variable_name = configure_proxy(self.name).sections[name].variable_name
|
150
|
+
if !section_already_exists && !self.respond_to?(variable_name)
|
151
|
+
attr_accessor variable_name
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def desc(description)
|
156
|
+
configure_proxy(self.name).desc(description)
|
157
|
+
end
|
158
|
+
|
159
|
+
def merged_configure_proxy
|
160
|
+
configurables = ancestors.reverse.select{ |a| a.respond_to?(:configure_proxy) }
|
161
|
+
|
162
|
+
# 'a.object_id.to_s' is to support anonymous class
|
163
|
+
# which created in tests to overwrite original behavior temporally
|
164
|
+
#
|
165
|
+
# p Module.new.name #=> nil
|
166
|
+
# p Class.new.name #=> nil
|
167
|
+
# p AnyGreatClass.dup.name #=> nil
|
168
|
+
configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
|
169
|
+
end
|
170
|
+
|
171
|
+
def dump(level = 0)
|
172
|
+
configure_proxy_map[self.to_s].dump(level)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
|
4
|
+
here = File.dirname(__FILE__)
|
5
|
+
$LOAD_PATH << File.expand_path(File.join(here, '..'))
|
6
|
+
|
7
|
+
require 'serverengine'
|
8
|
+
require 'fluent/supervisor'
|
9
|
+
|
10
|
+
server_module = Fluent.const_get(ARGV[0])
|
11
|
+
worker_module = Fluent.const_get(ARGV[1])
|
12
|
+
# it doesn't call ARGV in block because when reloading config, params will be initialized and then it can't use previous config.
|
13
|
+
config_path = ARGV[2]
|
14
|
+
params = JSON.parse(ARGV[3])
|
15
|
+
ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }
|
@@ -0,0 +1,220 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'socket'
|
18
|
+
|
19
|
+
require 'cool.io'
|
20
|
+
|
21
|
+
require 'fluent/config'
|
22
|
+
require 'fluent/event'
|
23
|
+
require 'fluent/event_router'
|
24
|
+
require 'fluent/msgpack_factory'
|
25
|
+
require 'fluent/root_agent'
|
26
|
+
require 'fluent/time'
|
27
|
+
require 'fluent/system_config'
|
28
|
+
require 'fluent/plugin'
|
29
|
+
|
30
|
+
module Fluent
|
31
|
+
class EngineClass
|
32
|
+
include Fluent::MessagePackFactory::Mixin
|
33
|
+
|
34
|
+
def initialize
|
35
|
+
@root_agent = nil
|
36
|
+
@event_router = nil
|
37
|
+
@default_loop = nil
|
38
|
+
@engine_stopped = false
|
39
|
+
|
40
|
+
@log_emit_thread = nil
|
41
|
+
@log_event_loop_stop = false
|
42
|
+
@log_event_queue = []
|
43
|
+
|
44
|
+
@suppress_config_dump = false
|
45
|
+
|
46
|
+
@system_config = SystemConfig.new
|
47
|
+
end
|
48
|
+
|
49
|
+
MAINLOOP_SLEEP_INTERVAL = 0.3
|
50
|
+
|
51
|
+
MATCH_CACHE_SIZE = 1024
|
52
|
+
LOG_EMIT_INTERVAL = 0.1
|
53
|
+
|
54
|
+
attr_reader :root_agent
|
55
|
+
attr_reader :matches, :sources
|
56
|
+
attr_reader :system_config
|
57
|
+
|
58
|
+
def init(system_config)
|
59
|
+
@system_config = system_config
|
60
|
+
|
61
|
+
BasicSocket.do_not_reverse_lookup = true
|
62
|
+
|
63
|
+
suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
|
64
|
+
@suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
|
65
|
+
@without_source = system_config.without_source unless system_config.without_source.nil?
|
66
|
+
|
67
|
+
@root_agent = RootAgent.new(log: log, system_config: @system_config)
|
68
|
+
|
69
|
+
MessagePackFactory.init
|
70
|
+
|
71
|
+
self
|
72
|
+
end
|
73
|
+
|
74
|
+
def log
|
75
|
+
$log
|
76
|
+
end
|
77
|
+
|
78
|
+
def suppress_interval(interval_time)
|
79
|
+
@suppress_emit_error_log_interval = interval_time
|
80
|
+
@next_emit_error_log_time = Time.now.to_i
|
81
|
+
end
|
82
|
+
|
83
|
+
def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
|
84
|
+
if fname =~ /\.rb$/
|
85
|
+
require 'fluent/config/dsl'
|
86
|
+
Config::DSL::Parser.parse(io, File.join(basepath, fname))
|
87
|
+
else
|
88
|
+
Config.parse(io, fname, basepath, v1_config)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def run_configure(conf)
|
93
|
+
configure(conf)
|
94
|
+
conf.check_not_fetched { |key, e|
|
95
|
+
parent_name, plugin_name = e.unused_in
|
96
|
+
if parent_name
|
97
|
+
message = if plugin_name
|
98
|
+
"section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
|
99
|
+
else
|
100
|
+
"section <#{e.name}> is not used in <#{parent_name}>"
|
101
|
+
end
|
102
|
+
$log.warn message
|
103
|
+
next
|
104
|
+
end
|
105
|
+
unless e.name == 'system'
|
106
|
+
unless @without_source && e.name == 'source'
|
107
|
+
$log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
|
108
|
+
end
|
109
|
+
end
|
110
|
+
}
|
111
|
+
end
|
112
|
+
|
113
|
+
def configure(conf)
|
114
|
+
# plugins / configuration dumps
|
115
|
+
Gem::Specification.find_all.select{|x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/}.each do |spec|
|
116
|
+
$log.info "gem '#{spec.name}' version '#{spec.version}'"
|
117
|
+
end
|
118
|
+
|
119
|
+
@root_agent.configure(conf)
|
120
|
+
@event_router = @root_agent.event_router
|
121
|
+
|
122
|
+
unless @suppress_config_dump
|
123
|
+
$log.info "using configuration file: #{conf.to_s.rstrip}"
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def add_plugin_dir(dir)
|
128
|
+
Plugin.add_plugin_dir(dir)
|
129
|
+
end
|
130
|
+
|
131
|
+
def emit(tag, time, record)
|
132
|
+
raise "BUG: use router.emit instead of Engine.emit"
|
133
|
+
end
|
134
|
+
|
135
|
+
def emit_array(tag, array)
|
136
|
+
raise "BUG: use router.emit_array instead of Engine.emit_array"
|
137
|
+
end
|
138
|
+
|
139
|
+
def emit_stream(tag, es)
|
140
|
+
raise "BUG: use router.emit_stream instead of Engine.emit_stream"
|
141
|
+
end
|
142
|
+
|
143
|
+
def flush!
|
144
|
+
@root_agent.flush!
|
145
|
+
end
|
146
|
+
|
147
|
+
def now
|
148
|
+
# TODO thread update
|
149
|
+
Fluent::EventTime.now
|
150
|
+
end
|
151
|
+
|
152
|
+
def log_event_loop
|
153
|
+
$log.disable_events(Thread.current)
|
154
|
+
|
155
|
+
while sleep(LOG_EMIT_INTERVAL)
|
156
|
+
break if @log_event_loop_stop
|
157
|
+
next if @log_event_queue.empty?
|
158
|
+
|
159
|
+
# NOTE: thead-safe of slice! depends on GVL
|
160
|
+
events = @log_event_queue.slice!(0..-1)
|
161
|
+
next if events.empty?
|
162
|
+
|
163
|
+
events.each {|tag,time,record|
|
164
|
+
begin
|
165
|
+
@event_router.emit(tag, time, record)
|
166
|
+
rescue => e
|
167
|
+
$log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
|
168
|
+
end
|
169
|
+
}
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def run
|
174
|
+
begin
|
175
|
+
start
|
176
|
+
|
177
|
+
if @event_router.match?($log.tag)
|
178
|
+
$log.enable_event
|
179
|
+
@log_emit_thread = Thread.new(&method(:log_event_loop))
|
180
|
+
end
|
181
|
+
|
182
|
+
sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
|
183
|
+
|
184
|
+
rescue Exception => e
|
185
|
+
$log.error "unexpected error", error: e
|
186
|
+
$log.error_backtrace
|
187
|
+
raise
|
188
|
+
end
|
189
|
+
|
190
|
+
$log.info "shutting down fluentd"
|
191
|
+
shutdown
|
192
|
+
if @log_emit_thread
|
193
|
+
@log_event_loop_stop = true
|
194
|
+
@log_emit_thread.join
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
def stop
|
199
|
+
@engine_stopped = true
|
200
|
+
nil
|
201
|
+
end
|
202
|
+
|
203
|
+
def push_log_event(tag, time, record)
|
204
|
+
return if @log_emit_thread.nil?
|
205
|
+
@log_event_queue.push([tag, time, record])
|
206
|
+
end
|
207
|
+
|
208
|
+
private
|
209
|
+
|
210
|
+
def start
|
211
|
+
@root_agent.start
|
212
|
+
end
|
213
|
+
|
214
|
+
def shutdown
|
215
|
+
@root_agent.shutdown
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
Engine = EngineClass.new
|
220
|
+
end
|