fluentd 0.14.4-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +26 -0
- data/.travis.yml +45 -0
- data/AUTHORS +2 -0
- data/CONTRIBUTING.md +35 -0
- data/COPYING +14 -0
- data/ChangeLog +276 -0
- data/Gemfile +9 -0
- data/README.md +51 -0
- data/Rakefile +53 -0
- data/Vagrantfile +17 -0
- data/appveyor.yml +41 -0
- data/bin/fluent-debug +5 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +11 -0
- data/example/in_http.conf +14 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_buffered_null.conf +32 -0
- data/example/out_copy.conf +20 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +51 -0
- data/lib/fluent/agent.rb +194 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/cat.rb +319 -0
- data/lib/fluent/command/debug.rb +102 -0
- data/lib/fluent/command/fluentd.rb +273 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +677 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +180 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config.rb +56 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +366 -0
- data/lib/fluent/config/dsl.rb +149 -0
- data/lib/fluent/config/element.rb +218 -0
- data/lib/fluent/config/error.rb +26 -0
- data/lib/fluent/config/literal_parser.rb +251 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +212 -0
- data/lib/fluent/config/types.rb +136 -0
- data/lib/fluent/config/v1_parser.rb +190 -0
- data/lib/fluent/configurable.rb +176 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +220 -0
- data/lib/fluent/env.rb +27 -0
- data/lib/fluent/event.rb +287 -0
- data/lib/fluent/event_router.rb +259 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +38 -0
- data/lib/fluent/load.rb +36 -0
- data/lib/fluent/log.rb +445 -0
- data/lib/fluent/match.rb +141 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +26 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin.rb +161 -0
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +130 -0
- data/lib/fluent/plugin/buf_file.rb +154 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer.rb +603 -0
- data/lib/fluent/plugin/buffer/chunk.rb +160 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +75 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +45 -0
- data/lib/fluent/plugin/formatter_csv.rb +47 -0
- data/lib/fluent/plugin/formatter_hash.rb +29 -0
- data/lib/fluent/plugin/formatter_json.rb +44 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
- data/lib/fluent/plugin/formatter_out_file.rb +78 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +74 -0
- data/lib/fluent/plugin/in_debug_agent.rb +64 -0
- data/lib/fluent/plugin/in_dummy.rb +135 -0
- data/lib/fluent/plugin/in_exec.rb +149 -0
- data/lib/fluent/plugin/in_forward.rb +366 -0
- data/lib/fluent/plugin/in_gc_stat.rb +52 -0
- data/lib/fluent/plugin/in_http.rb +422 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
- data/lib/fluent/plugin/in_object_space.rb +90 -0
- data/lib/fluent/plugin/in_syslog.rb +204 -0
- data/lib/fluent/plugin/in_tail.rb +838 -0
- data/lib/fluent/plugin/in_tcp.rb +41 -0
- data/lib/fluent/plugin/in_udp.rb +37 -0
- data/lib/fluent/plugin/in_unix.rb +201 -0
- data/lib/fluent/plugin/input.rb +33 -0
- data/lib/fluent/plugin/multi_output.rb +95 -0
- data/lib/fluent/plugin/out_buffered_null.rb +59 -0
- data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
- data/lib/fluent/plugin/out_copy.rb +42 -0
- data/lib/fluent/plugin/out_exec.rb +114 -0
- data/lib/fluent/plugin/out_exec_filter.rb +393 -0
- data/lib/fluent/plugin/out_file.rb +167 -0
- data/lib/fluent/plugin/out_forward.rb +646 -0
- data/lib/fluent/plugin/out_null.rb +27 -0
- data/lib/fluent/plugin/out_relabel.rb +28 -0
- data/lib/fluent/plugin/out_roundrobin.rb +80 -0
- data/lib/fluent/plugin/out_stdout.rb +48 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1020 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +175 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +33 -0
- data/lib/fluent/plugin/parser_json.rb +79 -0
- data/lib/fluent/plugin/parser_ltsv.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +104 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +73 -0
- data/lib/fluent/plugin/parser_syslog.rb +82 -0
- data/lib/fluent/plugin/parser_tsv.rb +37 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +132 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin_helper.rb +42 -0
- data/lib/fluent/plugin_helper/child_process.rb +298 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
- data/lib/fluent/plugin_helper/event_loop.rb +118 -0
- data/lib/fluent/plugin_helper/formatter.rb +149 -0
- data/lib/fluent/plugin_helper/inject.rb +125 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/retry_state.rb +177 -0
- data/lib/fluent/plugin_helper/storage.rb +331 -0
- data/lib/fluent/plugin_helper/thread.rb +147 -0
- data/lib/fluent/plugin_helper/timer.rb +90 -0
- data/lib/fluent/plugin_id.rb +63 -0
- data/lib/fluent/process.rb +504 -0
- data/lib/fluent/registry.rb +99 -0
- data/lib/fluent/root_agent.rb +314 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/supervisor.rb +680 -0
- data/lib/fluent/system_config.rb +122 -0
- data/lib/fluent/test.rb +56 -0
- data/lib/fluent/test/base.rb +85 -0
- data/lib/fluent/test/driver/base.rb +179 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +125 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +52 -0
- data/lib/fluent/test/driver/output.rb +76 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +79 -0
- data/lib/fluent/test/input_test.rb +172 -0
- data/lib/fluent/test/log.rb +73 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/time.rb +175 -0
- data/lib/fluent/timezone.rb +133 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +71 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +82 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +507 -0
- data/test/config/test_configurable.rb +1194 -0
- data/test/config/test_configure_proxy.rb +386 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +403 -0
- data/test/config/test_literal_parser.rb +297 -0
- data/test/config/test_section.rb +184 -0
- data/test/config/test_system_config.rb +120 -0
- data/test/config/test_types.rb +171 -0
- data/test/helper.rb +119 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +75 -0
- data/test/plugin/test_buf_file.rb +571 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1200 -0
- data/test/plugin/test_buffer_chunk.rb +168 -0
- data/test/plugin/test_buffer_file_chunk.rb +771 -0
- data/test/plugin/test_buffer_memory_chunk.rb +265 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_filter.rb +353 -0
- data/test/plugin/test_filter_grep.rb +119 -0
- data/test/plugin/test_filter_record_transformer.rb +600 -0
- data/test/plugin/test_filter_stdout.rb +211 -0
- data/test/plugin/test_formatter_csv.rb +94 -0
- data/test/plugin/test_formatter_json.rb +30 -0
- data/test/plugin/test_formatter_ltsv.rb +52 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_in_debug_agent.rb +28 -0
- data/test/plugin/test_in_dummy.rb +188 -0
- data/test/plugin/test_in_exec.rb +133 -0
- data/test/plugin/test_in_forward.rb +635 -0
- data/test/plugin/test_in_gc_stat.rb +39 -0
- data/test/plugin/test_in_http.rb +442 -0
- data/test/plugin/test_in_monitor_agent.rb +329 -0
- data/test/plugin/test_in_object_space.rb +64 -0
- data/test/plugin/test_in_syslog.rb +205 -0
- data/test/plugin/test_in_tail.rb +1001 -0
- data/test/plugin/test_in_tcp.rb +102 -0
- data/test/plugin/test_in_udp.rb +121 -0
- data/test/plugin/test_in_unix.rb +126 -0
- data/test/plugin/test_input.rb +122 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_buffered_null.rb +79 -0
- data/test/plugin/test_out_buffered_stdout.rb +122 -0
- data/test/plugin/test_out_copy.rb +160 -0
- data/test/plugin/test_out_exec.rb +155 -0
- data/test/plugin/test_out_exec_filter.rb +262 -0
- data/test/plugin/test_out_file.rb +383 -0
- data/test/plugin/test_out_forward.rb +590 -0
- data/test/plugin/test_out_null.rb +29 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_stdout.rb +92 -0
- data/test/plugin/test_out_stream.rb +93 -0
- data/test/plugin/test_output.rb +568 -0
- data/test/plugin/test_output_as_buffered.rb +1604 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +839 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +38 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_base.rb +32 -0
- data/test/plugin/test_parser_csv.rb +104 -0
- data/test/plugin/test_parser_json.rb +107 -0
- data/test/plugin/test_parser_labeled_tsv.rb +129 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +48 -0
- data/test/plugin/test_parser_none.rb +53 -0
- data/test/plugin/test_parser_regexp.rb +277 -0
- data/test/plugin/test_parser_syslog.rb +66 -0
- data/test/plugin/test_parser_time.rb +46 -0
- data/test/plugin/test_parser_tsv.rb +121 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +8 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/test_child_process.rb +608 -0
- data/test/plugin_helper/test_compat_parameters.rb +242 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_formatter.rb +252 -0
- data/test/plugin_helper/test_inject.rb +487 -0
- data/test/plugin_helper/test_parser.rb +263 -0
- data/test/plugin_helper/test_retry_state.rb +399 -0
- data/test/plugin_helper/test_storage.rb +521 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +131 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_config.rb +179 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_event.rb +329 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +184 -0
- data/test/test_filter.rb +121 -0
- data/test/test_formatter.rb +319 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +572 -0
- data/test/test_match.rb +137 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_output.rb +214 -0
- data/test/test_plugin_classes.rb +136 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_process.rb +48 -0
- data/test/test_root_agent.rb +278 -0
- data/test/test_supervisor.rb +339 -0
- data/test/test_time_formatter.rb +186 -0
- data/test/test_unique_id.rb +47 -0
- metadata +823 -0
data/lib/fluent/env.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
|
19
|
+
DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
|
20
|
+
DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
|
21
|
+
IS_WINDOWS = /mswin|mingw/ === RUBY_PLATFORM
|
22
|
+
private_constant :IS_WINDOWS
|
23
|
+
|
24
|
+
def self.windows?
|
25
|
+
IS_WINDOWS
|
26
|
+
end
|
27
|
+
end
|
data/lib/fluent/event.rb
ADDED
@@ -0,0 +1,287 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/msgpack_factory'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
class EventStream
|
21
|
+
include Enumerable
|
22
|
+
include MessagePackFactory::Mixin
|
23
|
+
|
24
|
+
# dup does deep copy for event stream
|
25
|
+
def dup
|
26
|
+
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
27
|
+
end
|
28
|
+
|
29
|
+
def size
|
30
|
+
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
31
|
+
end
|
32
|
+
alias :length :size
|
33
|
+
|
34
|
+
def empty?
|
35
|
+
size == 0
|
36
|
+
end
|
37
|
+
|
38
|
+
# for tests
|
39
|
+
def ==(other)
|
40
|
+
other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream
|
41
|
+
end
|
42
|
+
|
43
|
+
def repeatable?
|
44
|
+
false
|
45
|
+
end
|
46
|
+
|
47
|
+
def slice(index, num)
|
48
|
+
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
49
|
+
end
|
50
|
+
|
51
|
+
def each(&block)
|
52
|
+
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
53
|
+
end
|
54
|
+
|
55
|
+
def to_msgpack_stream(time_int: false)
|
56
|
+
return to_msgpack_stream_forced_integer if time_int
|
57
|
+
out = msgpack_packer
|
58
|
+
each {|time,record|
|
59
|
+
out.write([time,record])
|
60
|
+
}
|
61
|
+
out.to_s
|
62
|
+
end
|
63
|
+
|
64
|
+
def to_msgpack_stream_forced_integer
|
65
|
+
out = msgpack_packer
|
66
|
+
each {|time,record|
|
67
|
+
out.write([time.to_i,record])
|
68
|
+
}
|
69
|
+
out.to_s
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
class OneEventStream < EventStream
|
74
|
+
def initialize(time, record)
|
75
|
+
@time = time
|
76
|
+
@record = record
|
77
|
+
end
|
78
|
+
|
79
|
+
def dup
|
80
|
+
OneEventStream.new(@time, @record.dup)
|
81
|
+
end
|
82
|
+
|
83
|
+
def size
|
84
|
+
1
|
85
|
+
end
|
86
|
+
|
87
|
+
def repeatable?
|
88
|
+
true
|
89
|
+
end
|
90
|
+
|
91
|
+
def slice(index, num)
|
92
|
+
if index > 0 || num == 0
|
93
|
+
ArrayEventStream.new([])
|
94
|
+
else
|
95
|
+
self.dup
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def each(&block)
|
100
|
+
block.call(@time, @record)
|
101
|
+
nil
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
# EventStream from entries: Array of [time, record]
|
106
|
+
#
|
107
|
+
# Use this class for many events data with a tag
|
108
|
+
# and its representation is [ [time, record], [time, record], .. ]
|
109
|
+
class ArrayEventStream < EventStream
|
110
|
+
def initialize(entries)
|
111
|
+
@entries = entries
|
112
|
+
end
|
113
|
+
|
114
|
+
def dup
|
115
|
+
entries = @entries.map{ |time, record| [time, record.dup] }
|
116
|
+
ArrayEventStream.new(entries)
|
117
|
+
end
|
118
|
+
|
119
|
+
def size
|
120
|
+
@entries.size
|
121
|
+
end
|
122
|
+
|
123
|
+
def repeatable?
|
124
|
+
true
|
125
|
+
end
|
126
|
+
|
127
|
+
def empty?
|
128
|
+
@entries.empty?
|
129
|
+
end
|
130
|
+
|
131
|
+
def slice(index, num)
|
132
|
+
ArrayEventStream.new(@entries.slice(index, num))
|
133
|
+
end
|
134
|
+
|
135
|
+
def each(&block)
|
136
|
+
@entries.each(&block)
|
137
|
+
nil
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
# EventStream from entries: numbers of pairs of time and record.
|
142
|
+
#
|
143
|
+
# This class can handle many events more efficiently than ArrayEventStream
|
144
|
+
# because this class generate less objects than ArrayEventStream.
|
145
|
+
#
|
146
|
+
# Use this class as below, in loop of data-enumeration:
|
147
|
+
# 1. initialize blank stream:
|
148
|
+
# streams[tag] ||= MultiEventStream.new
|
149
|
+
# 2. add events
|
150
|
+
# stream[tag].add(time, record)
|
151
|
+
class MultiEventStream < EventStream
|
152
|
+
def initialize(time_array = [], record_array = [])
|
153
|
+
@time_array = time_array
|
154
|
+
@record_array = record_array
|
155
|
+
end
|
156
|
+
|
157
|
+
def dup
|
158
|
+
MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))
|
159
|
+
end
|
160
|
+
|
161
|
+
def size
|
162
|
+
@time_array.size
|
163
|
+
end
|
164
|
+
|
165
|
+
def add(time, record)
|
166
|
+
@time_array << time
|
167
|
+
@record_array << record
|
168
|
+
end
|
169
|
+
|
170
|
+
def repeatable?
|
171
|
+
true
|
172
|
+
end
|
173
|
+
|
174
|
+
def empty?
|
175
|
+
@time_array.empty?
|
176
|
+
end
|
177
|
+
|
178
|
+
def slice(index, num)
|
179
|
+
MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
|
180
|
+
end
|
181
|
+
|
182
|
+
def each(&block)
|
183
|
+
time_array = @time_array
|
184
|
+
record_array = @record_array
|
185
|
+
for i in 0..time_array.length-1
|
186
|
+
block.call(time_array[i], record_array[i])
|
187
|
+
end
|
188
|
+
nil
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
class MessagePackEventStream < EventStream
|
193
|
+
# https://github.com/msgpack/msgpack-ruby/issues/119
|
194
|
+
|
195
|
+
# Keep cached_unpacker argument for existing plugins
|
196
|
+
def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
|
197
|
+
@data = data
|
198
|
+
@size = size
|
199
|
+
@unpacked_times = unpacked_times
|
200
|
+
@unpacked_records = unpacked_records
|
201
|
+
end
|
202
|
+
|
203
|
+
def empty?
|
204
|
+
@data.empty?
|
205
|
+
end
|
206
|
+
|
207
|
+
def dup
|
208
|
+
if @unpacked_times
|
209
|
+
MessagePackEventStream.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
|
210
|
+
else
|
211
|
+
MessagePackEventStream.new(@data.dup, nil, @size)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
def size
|
216
|
+
# @size is unbelievable always when @size == 0
|
217
|
+
# If the number of events is really zero, unpacking events takes very short time.
|
218
|
+
ensure_unpacked! if @size == 0
|
219
|
+
@size
|
220
|
+
end
|
221
|
+
|
222
|
+
def repeatable?
|
223
|
+
true
|
224
|
+
end
|
225
|
+
|
226
|
+
def ensure_unpacked!
|
227
|
+
return if @unpacked_times && @unpacked_records
|
228
|
+
@unpacked_times = []
|
229
|
+
@unpacked_records = []
|
230
|
+
msgpack_unpacker.feed_each(@data) do |time, record|
|
231
|
+
@unpacked_times << time
|
232
|
+
@unpacked_records << record
|
233
|
+
end
|
234
|
+
# @size should be updated always right after unpack.
|
235
|
+
# The real size of unpacked objects are correct, rather than given size.
|
236
|
+
@size = @unpacked_times.size
|
237
|
+
end
|
238
|
+
|
239
|
+
# This method returns MultiEventStream, because there are no reason
|
240
|
+
# to surve binary serialized by msgpack.
|
241
|
+
def slice(index, num)
|
242
|
+
ensure_unpacked!
|
243
|
+
MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
|
244
|
+
end
|
245
|
+
|
246
|
+
def each(&block)
|
247
|
+
if @unpacked_times
|
248
|
+
@unpacked_times.each_with_index do |time, i|
|
249
|
+
block.call(time, @unpacked_records[i])
|
250
|
+
end
|
251
|
+
else
|
252
|
+
@unpacked_times = []
|
253
|
+
@unpacked_records = []
|
254
|
+
msgpack_unpacker.feed_each(@data) do |time, record|
|
255
|
+
@unpacked_times << time
|
256
|
+
@unpacked_records << record
|
257
|
+
block.call(time, record)
|
258
|
+
end
|
259
|
+
@size = @unpacked_times.size
|
260
|
+
end
|
261
|
+
nil
|
262
|
+
end
|
263
|
+
|
264
|
+
def to_msgpack_stream(time_int: false)
|
265
|
+
# time_int is always ignored because @data is always packed binary in this class
|
266
|
+
@data
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
module ChunkMessagePackEventStreamer
|
271
|
+
include MessagePackFactory::Mixin
|
272
|
+
# chunk.extend(ChunkEventStreamer)
|
273
|
+
# => chunk.each{|time, record| ... }
|
274
|
+
def each(&block)
|
275
|
+
open do |io|
|
276
|
+
msgpack_unpacker(io).each(&block)
|
277
|
+
end
|
278
|
+
nil
|
279
|
+
end
|
280
|
+
alias :msgpack_each :each
|
281
|
+
|
282
|
+
def to_msgpack_stream(time_int: false)
|
283
|
+
# time_int is always ignored because data is already packed and written in chunk
|
284
|
+
read
|
285
|
+
end
|
286
|
+
end
|
287
|
+
end
|
@@ -0,0 +1,259 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/match'
|
18
|
+
require 'fluent/event'
|
19
|
+
require 'fluent/filter'
|
20
|
+
|
21
|
+
module Fluent
|
22
|
+
#
|
23
|
+
# EventRouter is responsible to route events to a collector.
|
24
|
+
#
|
25
|
+
# It has a list of MatchPattern and Collector pairs:
|
26
|
+
#
|
27
|
+
# +----------------+ +-----------------+
|
28
|
+
# | MatchPattern | | Collector |
|
29
|
+
# +----------------+ +-----------------+
|
30
|
+
# | access.** ---------> type forward |
|
31
|
+
# | logs.** ---------> type copy |
|
32
|
+
# | archive.** ---------> type s3 |
|
33
|
+
# +----------------+ +-----------------+
|
34
|
+
#
|
35
|
+
# EventRouter does:
|
36
|
+
#
|
37
|
+
# 1) receive an event at `#emit` methods
|
38
|
+
# 2) match the event's tag with the MatchPatterns
|
39
|
+
# 3) forward the event to the corresponding Collector
|
40
|
+
#
|
41
|
+
# Collector is either of Output, Filter or other EventRouter.
|
42
|
+
#
|
43
|
+
class EventRouter
|
44
|
+
def initialize(default_collector, emit_error_handler)
|
45
|
+
@match_rules = []
|
46
|
+
@match_cache = MatchCache.new
|
47
|
+
@default_collector = default_collector
|
48
|
+
@emit_error_handler = emit_error_handler
|
49
|
+
end
|
50
|
+
|
51
|
+
attr_accessor :default_collector
|
52
|
+
attr_accessor :emit_error_handler
|
53
|
+
|
54
|
+
class Rule
|
55
|
+
def initialize(pattern, collector)
|
56
|
+
patterns = pattern.split(/\s+/).map { |str| MatchPattern.create(str) }
|
57
|
+
@pattern = if patterns.length == 1
|
58
|
+
patterns[0]
|
59
|
+
else
|
60
|
+
OrMatchPattern.new(patterns)
|
61
|
+
end
|
62
|
+
@pattern_str = pattern
|
63
|
+
@collector = collector
|
64
|
+
end
|
65
|
+
|
66
|
+
def match?(tag)
|
67
|
+
@pattern.match(tag)
|
68
|
+
end
|
69
|
+
|
70
|
+
attr_reader :collector
|
71
|
+
attr_reader :pattern_str
|
72
|
+
end
|
73
|
+
|
74
|
+
# called by Agent to add new match pattern and collector
|
75
|
+
def add_rule(pattern, collector)
|
76
|
+
@match_rules << Rule.new(pattern, collector)
|
77
|
+
end
|
78
|
+
|
79
|
+
def emit(tag, time, record)
|
80
|
+
unless record.nil?
|
81
|
+
emit_stream(tag, OneEventStream.new(time, record))
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def emit_array(tag, array)
|
86
|
+
emit_stream(tag, ArrayEventStream.new(array))
|
87
|
+
end
|
88
|
+
|
89
|
+
def emit_stream(tag, es)
|
90
|
+
match(tag).emit_events(tag, es)
|
91
|
+
rescue => e
|
92
|
+
@emit_error_handler.handle_emits_error(tag, es, e)
|
93
|
+
end
|
94
|
+
|
95
|
+
def emit_error_event(tag, time, record, error)
|
96
|
+
@emit_error_handler.emit_error_event(tag, time, record, error)
|
97
|
+
end
|
98
|
+
|
99
|
+
def match?(tag)
|
100
|
+
!!find(tag)
|
101
|
+
end
|
102
|
+
|
103
|
+
def match(tag)
|
104
|
+
collector = @match_cache.get(tag) {
|
105
|
+
find(tag) || @default_collector
|
106
|
+
}
|
107
|
+
collector
|
108
|
+
end
|
109
|
+
|
110
|
+
class MatchCache
|
111
|
+
MATCH_CACHE_SIZE = 1024
|
112
|
+
|
113
|
+
def initialize
|
114
|
+
super
|
115
|
+
@map = {}
|
116
|
+
@keys = []
|
117
|
+
end
|
118
|
+
|
119
|
+
def get(key)
|
120
|
+
if collector = @map[key]
|
121
|
+
return collector
|
122
|
+
end
|
123
|
+
collector = @map[key] = yield
|
124
|
+
if @keys.size >= MATCH_CACHE_SIZE
|
125
|
+
# expire the oldest key
|
126
|
+
@map.delete @keys.shift
|
127
|
+
end
|
128
|
+
@keys << key
|
129
|
+
collector
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
private
|
134
|
+
|
135
|
+
class Pipeline
|
136
|
+
def initialize
|
137
|
+
@filters = []
|
138
|
+
@output = nil
|
139
|
+
@optimizer = FilterOptimizer.new
|
140
|
+
end
|
141
|
+
|
142
|
+
def add_filter(filter)
|
143
|
+
@filters << filter
|
144
|
+
@optimizer.filters = @filters
|
145
|
+
end
|
146
|
+
|
147
|
+
def set_output(output)
|
148
|
+
@output = output
|
149
|
+
end
|
150
|
+
|
151
|
+
def emit_events(tag, es)
|
152
|
+
processed = @optimizer.filter_stream(tag, es)
|
153
|
+
@output.emit_events(tag, processed)
|
154
|
+
end
|
155
|
+
|
156
|
+
class FilterOptimizer
|
157
|
+
def initialize(filters = [])
|
158
|
+
@filters = filters
|
159
|
+
@optimizable = nil
|
160
|
+
end
|
161
|
+
|
162
|
+
def filters=(filters)
|
163
|
+
@filters = filters
|
164
|
+
reset_optimization
|
165
|
+
end
|
166
|
+
|
167
|
+
def filter_stream(tag, es)
|
168
|
+
if optimizable?
|
169
|
+
optimized_filter_stream(tag, es)
|
170
|
+
else
|
171
|
+
@filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
private
|
176
|
+
|
177
|
+
def optimized_filter_stream(tag, es)
|
178
|
+
new_es = MultiEventStream.new
|
179
|
+
es.each do |time, record|
|
180
|
+
filtered_record = record
|
181
|
+
filtered_time = time
|
182
|
+
|
183
|
+
catch :break_loop do
|
184
|
+
@filters.each do |filter|
|
185
|
+
if filter.has_filter_with_time
|
186
|
+
begin
|
187
|
+
filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
|
188
|
+
throw :break_loop unless filtered_record && filtered_time
|
189
|
+
rescue => e
|
190
|
+
filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
|
191
|
+
end
|
192
|
+
else
|
193
|
+
begin
|
194
|
+
filtered_record = filter.filter(tag, filtered_time, filtered_record)
|
195
|
+
throw :break_loop unless filtered_record
|
196
|
+
rescue => e
|
197
|
+
filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
new_es.add(filtered_time, filtered_record)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
new_es
|
206
|
+
end
|
207
|
+
|
208
|
+
def optimizable?
|
209
|
+
return @optimizable unless @optimizable.nil?
|
210
|
+
@optimizable = if filters_having_filter_stream.empty?
|
211
|
+
true
|
212
|
+
else
|
213
|
+
$log.info "Filtering works with worse performance, because #{filters_having_filter_stream.map(&:class)} uses `#filter_stream` method."
|
214
|
+
false
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
def filters_having_filter_stream
|
219
|
+
@filters_having_filter_stream ||= @filters.select do |filter|
|
220
|
+
filter.class.instance_methods(false).include?(:filter_stream)
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
def reset_optimization
|
225
|
+
@optimizable = nil
|
226
|
+
@filters_having_filter_stream = nil
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
def find(tag)
|
232
|
+
pipeline = nil
|
233
|
+
@match_rules.each_with_index { |rule, i|
|
234
|
+
if rule.match?(tag)
|
235
|
+
if rule.collector.is_a?(Plugin::Filter)
|
236
|
+
pipeline ||= Pipeline.new
|
237
|
+
pipeline.add_filter(rule.collector)
|
238
|
+
else
|
239
|
+
if pipeline
|
240
|
+
pipeline.set_output(rule.collector)
|
241
|
+
else
|
242
|
+
# Use Output directly when filter is not matched
|
243
|
+
pipeline = rule.collector
|
244
|
+
end
|
245
|
+
return pipeline
|
246
|
+
end
|
247
|
+
end
|
248
|
+
}
|
249
|
+
|
250
|
+
if pipeline
|
251
|
+
# filter is matched but no match
|
252
|
+
pipeline.set_output(@default_collector)
|
253
|
+
pipeline
|
254
|
+
else
|
255
|
+
nil
|
256
|
+
end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|