fluentd 0.14.4-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.github/ISSUE_TEMPLATE.md +6 -0
- data/.gitignore +26 -0
- data/.travis.yml +45 -0
- data/AUTHORS +2 -0
- data/CONTRIBUTING.md +35 -0
- data/COPYING +14 -0
- data/ChangeLog +276 -0
- data/Gemfile +9 -0
- data/README.md +51 -0
- data/Rakefile +53 -0
- data/Vagrantfile +17 -0
- data/appveyor.yml +41 -0
- data/bin/fluent-debug +5 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +11 -0
- data/example/in_http.conf +14 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_buffered_null.conf +32 -0
- data/example/out_copy.conf +20 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +51 -0
- data/lib/fluent/agent.rb +194 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/cat.rb +319 -0
- data/lib/fluent/command/debug.rb +102 -0
- data/lib/fluent/command/fluentd.rb +273 -0
- data/lib/fluent/compat/call_super_mixin.rb +67 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +677 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +180 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config.rb +56 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +366 -0
- data/lib/fluent/config/dsl.rb +149 -0
- data/lib/fluent/config/element.rb +218 -0
- data/lib/fluent/config/error.rb +26 -0
- data/lib/fluent/config/literal_parser.rb +251 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +212 -0
- data/lib/fluent/config/types.rb +136 -0
- data/lib/fluent/config/v1_parser.rb +190 -0
- data/lib/fluent/configurable.rb +176 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/engine.rb +220 -0
- data/lib/fluent/env.rb +27 -0
- data/lib/fluent/event.rb +287 -0
- data/lib/fluent/event_router.rb +259 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +38 -0
- data/lib/fluent/load.rb +36 -0
- data/lib/fluent/log.rb +445 -0
- data/lib/fluent/match.rb +141 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +62 -0
- data/lib/fluent/output.rb +26 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin.rb +161 -0
- data/lib/fluent/plugin/bare_output.rb +63 -0
- data/lib/fluent/plugin/base.rb +130 -0
- data/lib/fluent/plugin/buf_file.rb +154 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer.rb +603 -0
- data/lib/fluent/plugin/buffer/chunk.rb +160 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +120 -0
- data/lib/fluent/plugin/filter.rb +93 -0
- data/lib/fluent/plugin/filter_grep.rb +75 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +45 -0
- data/lib/fluent/plugin/formatter_csv.rb +47 -0
- data/lib/fluent/plugin/formatter_hash.rb +29 -0
- data/lib/fluent/plugin/formatter_json.rb +44 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
- data/lib/fluent/plugin/formatter_out_file.rb +78 -0
- data/lib/fluent/plugin/formatter_single_value.rb +34 -0
- data/lib/fluent/plugin/formatter_stdout.rb +74 -0
- data/lib/fluent/plugin/in_debug_agent.rb +64 -0
- data/lib/fluent/plugin/in_dummy.rb +135 -0
- data/lib/fluent/plugin/in_exec.rb +149 -0
- data/lib/fluent/plugin/in_forward.rb +366 -0
- data/lib/fluent/plugin/in_gc_stat.rb +52 -0
- data/lib/fluent/plugin/in_http.rb +422 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
- data/lib/fluent/plugin/in_object_space.rb +90 -0
- data/lib/fluent/plugin/in_syslog.rb +204 -0
- data/lib/fluent/plugin/in_tail.rb +838 -0
- data/lib/fluent/plugin/in_tcp.rb +41 -0
- data/lib/fluent/plugin/in_udp.rb +37 -0
- data/lib/fluent/plugin/in_unix.rb +201 -0
- data/lib/fluent/plugin/input.rb +33 -0
- data/lib/fluent/plugin/multi_output.rb +95 -0
- data/lib/fluent/plugin/out_buffered_null.rb +59 -0
- data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
- data/lib/fluent/plugin/out_copy.rb +42 -0
- data/lib/fluent/plugin/out_exec.rb +114 -0
- data/lib/fluent/plugin/out_exec_filter.rb +393 -0
- data/lib/fluent/plugin/out_file.rb +167 -0
- data/lib/fluent/plugin/out_forward.rb +646 -0
- data/lib/fluent/plugin/out_null.rb +27 -0
- data/lib/fluent/plugin/out_relabel.rb +28 -0
- data/lib/fluent/plugin/out_roundrobin.rb +80 -0
- data/lib/fluent/plugin/out_stdout.rb +48 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1020 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +175 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +84 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +33 -0
- data/lib/fluent/plugin/parser_json.rb +79 -0
- data/lib/fluent/plugin/parser_ltsv.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +104 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +73 -0
- data/lib/fluent/plugin/parser_syslog.rb +82 -0
- data/lib/fluent/plugin/parser_tsv.rb +37 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +132 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin_helper.rb +42 -0
- data/lib/fluent/plugin_helper/child_process.rb +298 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
- data/lib/fluent/plugin_helper/event_loop.rb +118 -0
- data/lib/fluent/plugin_helper/formatter.rb +149 -0
- data/lib/fluent/plugin_helper/inject.rb +125 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/retry_state.rb +177 -0
- data/lib/fluent/plugin_helper/storage.rb +331 -0
- data/lib/fluent/plugin_helper/thread.rb +147 -0
- data/lib/fluent/plugin_helper/timer.rb +90 -0
- data/lib/fluent/plugin_id.rb +63 -0
- data/lib/fluent/process.rb +504 -0
- data/lib/fluent/registry.rb +99 -0
- data/lib/fluent/root_agent.rb +314 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/supervisor.rb +680 -0
- data/lib/fluent/system_config.rb +122 -0
- data/lib/fluent/test.rb +56 -0
- data/lib/fluent/test/base.rb +85 -0
- data/lib/fluent/test/driver/base.rb +179 -0
- data/lib/fluent/test/driver/base_owned.rb +70 -0
- data/lib/fluent/test/driver/base_owner.rb +125 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +52 -0
- data/lib/fluent/test/driver/output.rb +76 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +79 -0
- data/lib/fluent/test/input_test.rb +172 -0
- data/lib/fluent/test/log.rb +73 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/time.rb +175 -0
- data/lib/fluent/timezone.rb +133 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +71 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +82 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +507 -0
- data/test/config/test_configurable.rb +1194 -0
- data/test/config/test_configure_proxy.rb +386 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +403 -0
- data/test/config/test_literal_parser.rb +297 -0
- data/test/config/test_section.rb +184 -0
- data/test/config/test_system_config.rb +120 -0
- data/test/config/test_types.rb +171 -0
- data/test/helper.rb +119 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/test_bare_output.rb +118 -0
- data/test/plugin/test_base.rb +75 -0
- data/test/plugin/test_buf_file.rb +571 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1200 -0
- data/test/plugin/test_buffer_chunk.rb +168 -0
- data/test/plugin/test_buffer_file_chunk.rb +771 -0
- data/test/plugin/test_buffer_memory_chunk.rb +265 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_filter.rb +353 -0
- data/test/plugin/test_filter_grep.rb +119 -0
- data/test/plugin/test_filter_record_transformer.rb +600 -0
- data/test/plugin/test_filter_stdout.rb +211 -0
- data/test/plugin/test_formatter_csv.rb +94 -0
- data/test/plugin/test_formatter_json.rb +30 -0
- data/test/plugin/test_formatter_ltsv.rb +52 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_in_debug_agent.rb +28 -0
- data/test/plugin/test_in_dummy.rb +188 -0
- data/test/plugin/test_in_exec.rb +133 -0
- data/test/plugin/test_in_forward.rb +635 -0
- data/test/plugin/test_in_gc_stat.rb +39 -0
- data/test/plugin/test_in_http.rb +442 -0
- data/test/plugin/test_in_monitor_agent.rb +329 -0
- data/test/plugin/test_in_object_space.rb +64 -0
- data/test/plugin/test_in_syslog.rb +205 -0
- data/test/plugin/test_in_tail.rb +1001 -0
- data/test/plugin/test_in_tcp.rb +102 -0
- data/test/plugin/test_in_udp.rb +121 -0
- data/test/plugin/test_in_unix.rb +126 -0
- data/test/plugin/test_input.rb +122 -0
- data/test/plugin/test_multi_output.rb +180 -0
- data/test/plugin/test_out_buffered_null.rb +79 -0
- data/test/plugin/test_out_buffered_stdout.rb +122 -0
- data/test/plugin/test_out_copy.rb +160 -0
- data/test/plugin/test_out_exec.rb +155 -0
- data/test/plugin/test_out_exec_filter.rb +262 -0
- data/test/plugin/test_out_file.rb +383 -0
- data/test/plugin/test_out_forward.rb +590 -0
- data/test/plugin/test_out_null.rb +29 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_stdout.rb +92 -0
- data/test/plugin/test_out_stream.rb +93 -0
- data/test/plugin/test_output.rb +568 -0
- data/test/plugin/test_output_as_buffered.rb +1604 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +839 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +38 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_base.rb +32 -0
- data/test/plugin/test_parser_csv.rb +104 -0
- data/test/plugin/test_parser_json.rb +107 -0
- data/test/plugin/test_parser_labeled_tsv.rb +129 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +48 -0
- data/test/plugin/test_parser_none.rb +53 -0
- data/test/plugin/test_parser_regexp.rb +277 -0
- data/test/plugin/test_parser_syslog.rb +66 -0
- data/test/plugin/test_parser_time.rb +46 -0
- data/test/plugin/test_parser_tsv.rb +121 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +8 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/test_child_process.rb +608 -0
- data/test/plugin_helper/test_compat_parameters.rb +242 -0
- data/test/plugin_helper/test_event_emitter.rb +51 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_formatter.rb +252 -0
- data/test/plugin_helper/test_inject.rb +487 -0
- data/test/plugin_helper/test_parser.rb +263 -0
- data/test/plugin_helper/test_retry_state.rb +399 -0
- data/test/plugin_helper/test_storage.rb +521 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +131 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_config.rb +179 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_event.rb +329 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +184 -0
- data/test/test_filter.rb +121 -0
- data/test/test_formatter.rb +319 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +572 -0
- data/test/test_match.rb +137 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_output.rb +214 -0
- data/test/test_plugin_classes.rb +136 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_process.rb +48 -0
- data/test/test_root_agent.rb +278 -0
- data/test/test_supervisor.rb +339 -0
- data/test/test_time_formatter.rb +186 -0
- data/test/test_unique_id.rb +47 -0
- metadata +823 -0
@@ -0,0 +1,90 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/buffer/chunk'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module Plugin
|
21
|
+
class Buffer
|
22
|
+
class MemoryChunk < Chunk
|
23
|
+
def initialize(metadata)
|
24
|
+
super
|
25
|
+
@chunk = ''.force_encoding(Encoding::ASCII_8BIT)
|
26
|
+
@chunk_bytes = 0
|
27
|
+
@adding_bytes = 0
|
28
|
+
@adding_size = 0
|
29
|
+
end
|
30
|
+
|
31
|
+
def concat(bulk, bulk_size)
|
32
|
+
raise "BUG: concatenating to unwritable chunk, now '#{self.state}'" unless self.writable?
|
33
|
+
|
34
|
+
bulk.force_encoding(Encoding::ASCII_8BIT)
|
35
|
+
@chunk << bulk
|
36
|
+
@adding_bytes += bulk.bytesize
|
37
|
+
@adding_size += bulk_size
|
38
|
+
true
|
39
|
+
end
|
40
|
+
|
41
|
+
def commit
|
42
|
+
@size += @adding_size
|
43
|
+
@chunk_bytes += @adding_bytes
|
44
|
+
|
45
|
+
@adding_bytes = @adding_size = 0
|
46
|
+
@modified_at = Time.now
|
47
|
+
true
|
48
|
+
end
|
49
|
+
|
50
|
+
def rollback
|
51
|
+
@chunk.slice!(@chunk_bytes, @adding_bytes)
|
52
|
+
@adding_bytes = @adding_size = 0
|
53
|
+
true
|
54
|
+
end
|
55
|
+
|
56
|
+
def bytesize
|
57
|
+
@chunk_bytes + @adding_bytes
|
58
|
+
end
|
59
|
+
|
60
|
+
def size
|
61
|
+
@size + @adding_size
|
62
|
+
end
|
63
|
+
|
64
|
+
def empty?
|
65
|
+
@chunk.empty?
|
66
|
+
end
|
67
|
+
|
68
|
+
def purge
|
69
|
+
super
|
70
|
+
@chunk = ''.force_encoding("ASCII-8BIT")
|
71
|
+
@chunk_bytes = @size = @adding_bytes = @adding_size = 0
|
72
|
+
true
|
73
|
+
end
|
74
|
+
|
75
|
+
def read
|
76
|
+
@chunk
|
77
|
+
end
|
78
|
+
|
79
|
+
def open(&block)
|
80
|
+
StringIO.open(@chunk, &block)
|
81
|
+
end
|
82
|
+
|
83
|
+
def write_to(io)
|
84
|
+
# re-implementation to optimize not to create StringIO
|
85
|
+
io.write @chunk
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/compat/exec_util'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
# obsolete
|
21
|
+
ExecUtil = Fluent::Compat::ExecUtil
|
22
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/compat/file_util'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
# obsolete
|
21
|
+
FileUtil = Fluent::Compat::FileUtil
|
22
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module FileWrapper
|
19
|
+
def self.open(*args)
|
20
|
+
io = WindowsFile.new(*args).io
|
21
|
+
if block_given?
|
22
|
+
v = yield io
|
23
|
+
io.close
|
24
|
+
v
|
25
|
+
else
|
26
|
+
io
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.stat(path)
|
31
|
+
f = WindowsFile.new(path)
|
32
|
+
s = f.stat
|
33
|
+
f.close
|
34
|
+
s
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
module WindowsFileExtension
|
39
|
+
attr_reader :path
|
40
|
+
|
41
|
+
def stat
|
42
|
+
s = super
|
43
|
+
s.instance_variable_set :@ino, @ino
|
44
|
+
def s.ino; @ino; end
|
45
|
+
s
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# To open and get stat with setting FILE_SHARE_DELETE
|
50
|
+
class WindowsFile
|
51
|
+
require 'windows/file'
|
52
|
+
require 'windows/error'
|
53
|
+
require 'windows/handle'
|
54
|
+
require 'windows/nio'
|
55
|
+
|
56
|
+
include Windows::Error
|
57
|
+
include Windows::File
|
58
|
+
include Windows::Handle
|
59
|
+
include Windows::NIO
|
60
|
+
|
61
|
+
def initialize(path, mode='r', sharemode=FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE)
|
62
|
+
@path = path
|
63
|
+
@file_handle = INVALID_HANDLE_VALUE
|
64
|
+
@mode = mode
|
65
|
+
|
66
|
+
|
67
|
+
access, creationdisposition, seektoend = case mode.delete('b')
|
68
|
+
when "r" ; [FILE_GENERIC_READ , OPEN_EXISTING, false]
|
69
|
+
when "r+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS , false]
|
70
|
+
when "w" ; [FILE_GENERIC_WRITE , CREATE_ALWAYS, false]
|
71
|
+
when "w+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, CREATE_ALWAYS, false]
|
72
|
+
when "a" ; [FILE_GENERIC_WRITE , OPEN_ALWAYS , true]
|
73
|
+
when "a+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS , true]
|
74
|
+
else raise "unknown mode '#{mode}'"
|
75
|
+
end
|
76
|
+
|
77
|
+
@file_handle = CreateFile.call(@path, access, sharemode,
|
78
|
+
0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
|
79
|
+
if @file_handle == INVALID_HANDLE_VALUE
|
80
|
+
err = GetLastError.call
|
81
|
+
if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
|
82
|
+
raise SystemCallError.new(2)
|
83
|
+
end
|
84
|
+
raise SystemCallError.new(err)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def close
|
89
|
+
CloseHandle.call(@file_handle)
|
90
|
+
@file_handle = INVALID_HANDLE_VALUE
|
91
|
+
end
|
92
|
+
|
93
|
+
def io
|
94
|
+
fd = _open_osfhandle(@file_handle, 0)
|
95
|
+
raise Errno::ENOENT if fd == -1
|
96
|
+
io = File.for_fd(fd, @mode)
|
97
|
+
io.instance_variable_set :@ino, self.ino
|
98
|
+
io.instance_variable_set :@path, @path
|
99
|
+
io.extend WindowsFileExtension
|
100
|
+
io
|
101
|
+
end
|
102
|
+
|
103
|
+
def ino
|
104
|
+
by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4) #72bytes
|
105
|
+
|
106
|
+
unless GetFileInformationByHandle.call(@file_handle, by_handle_file_information)
|
107
|
+
return 0
|
108
|
+
end
|
109
|
+
|
110
|
+
by_handle_file_information.unpack("I11Q1")[11] # fileindex
|
111
|
+
end
|
112
|
+
|
113
|
+
def stat
|
114
|
+
s = File.stat(@path)
|
115
|
+
s.instance_variable_set :@ino, self.ino
|
116
|
+
def s.ino; @ino; end
|
117
|
+
s
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end if Fluent.windows?
|
@@ -0,0 +1,93 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/base'
|
18
|
+
|
19
|
+
require 'fluent/event'
|
20
|
+
require 'fluent/log'
|
21
|
+
require 'fluent/plugin_id'
|
22
|
+
require 'fluent/plugin_helper'
|
23
|
+
|
24
|
+
module Fluent
|
25
|
+
module Plugin
|
26
|
+
class Filter < Base
|
27
|
+
include PluginId
|
28
|
+
include PluginLoggerMixin
|
29
|
+
include PluginHelper::Mixin
|
30
|
+
|
31
|
+
helpers :event_emitter
|
32
|
+
|
33
|
+
attr_reader :has_filter_with_time
|
34
|
+
|
35
|
+
def initialize
|
36
|
+
super
|
37
|
+
@has_filter_with_time = has_filter_with_time?
|
38
|
+
end
|
39
|
+
|
40
|
+
def filter(tag, time, record)
|
41
|
+
raise NotImplementedError, "BUG: filter plugins MUST implement this method"
|
42
|
+
end
|
43
|
+
|
44
|
+
def filter_with_time(tag, time, record)
|
45
|
+
raise NotImplementedError, "BUG: filter plugins MUST implement this method"
|
46
|
+
end
|
47
|
+
|
48
|
+
def filter_stream(tag, es)
|
49
|
+
new_es = MultiEventStream.new
|
50
|
+
if @has_filter_with_time
|
51
|
+
es.each do |time, record|
|
52
|
+
begin
|
53
|
+
filtered_time, filtered_record = filter_with_time(tag, time, record)
|
54
|
+
new_es.add(filtered_time, filtered_record) if filtered_time && filtered_record
|
55
|
+
rescue => e
|
56
|
+
router.emit_error_event(tag, time, record, e)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
else
|
60
|
+
es.each do |time, record|
|
61
|
+
begin
|
62
|
+
filtered_record = filter(tag, time, record)
|
63
|
+
new_es.add(time, filtered_record) if filtered_record
|
64
|
+
rescue => e
|
65
|
+
router.emit_error_event(tag, time, record, e)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
new_es
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
def has_filter_with_time?
|
75
|
+
implmented_methods = self.class.instance_methods(false)
|
76
|
+
# Plugins that override `filter_stream` don't need check,
|
77
|
+
# because they may not call `filter` or `filter_with_time`
|
78
|
+
# for example fluentd/lib/fluent/plugin/filter_record_transformer.rb
|
79
|
+
return nil if implmented_methods.include?(:filter_stream)
|
80
|
+
case
|
81
|
+
when [:filter, :filter_with_time].all? { |e| implmented_methods.include?(e) }
|
82
|
+
raise "BUG: Filter plugins MUST be implemented either `filter` or `filter_with_time`"
|
83
|
+
when implmented_methods.include?(:filter)
|
84
|
+
false
|
85
|
+
when implmented_methods.include?(:filter_with_time)
|
86
|
+
true
|
87
|
+
else
|
88
|
+
raise NotImplementedError, "BUG: Filter plugins MUST be implmented either `filter` or `filter_with_time`"
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/filter'
|
18
|
+
require 'fluent/config/error'
|
19
|
+
require 'fluent/plugin/string_util'
|
20
|
+
|
21
|
+
module Fluent::Plugin
|
22
|
+
class GrepFilter < Filter
|
23
|
+
Fluent::Plugin.register_filter('grep', self)
|
24
|
+
|
25
|
+
REGEXP_MAX_NUM = 20
|
26
|
+
|
27
|
+
(1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}", :string, default: nil }
|
28
|
+
(1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
|
29
|
+
|
30
|
+
# for test
|
31
|
+
attr_reader :regexps
|
32
|
+
attr_reader :excludes
|
33
|
+
|
34
|
+
def configure(conf)
|
35
|
+
super
|
36
|
+
|
37
|
+
@regexps = {}
|
38
|
+
(1..REGEXP_MAX_NUM).each do |i|
|
39
|
+
next unless conf["regexp#{i}"]
|
40
|
+
key, regexp = conf["regexp#{i}"].split(/ /, 2)
|
41
|
+
raise ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
|
42
|
+
raise ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
|
43
|
+
@regexps[key] = Regexp.compile(regexp)
|
44
|
+
end
|
45
|
+
|
46
|
+
@excludes = {}
|
47
|
+
(1..REGEXP_MAX_NUM).each do |i|
|
48
|
+
next unless conf["exclude#{i}"]
|
49
|
+
key, exclude = conf["exclude#{i}"].split(/ /, 2)
|
50
|
+
raise ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
|
51
|
+
raise ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
|
52
|
+
@excludes[key] = Regexp.compile(exclude)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def filter(tag, time, record)
|
57
|
+
result = nil
|
58
|
+
begin
|
59
|
+
catch(:break_loop) do
|
60
|
+
@regexps.each do |key, regexp|
|
61
|
+
throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
|
62
|
+
end
|
63
|
+
@excludes.each do |key, exclude|
|
64
|
+
throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
|
65
|
+
end
|
66
|
+
result = record
|
67
|
+
end
|
68
|
+
rescue => e
|
69
|
+
log.warn "failed to grep events", error: e
|
70
|
+
log.warn_backtrace
|
71
|
+
end
|
72
|
+
result
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,342 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'socket'
|
18
|
+
require 'json'
|
19
|
+
require 'ostruct'
|
20
|
+
|
21
|
+
require 'fluent/plugin/filter'
|
22
|
+
require 'fluent/config/error'
|
23
|
+
require 'fluent/event'
|
24
|
+
require 'fluent/time'
|
25
|
+
|
26
|
+
module Fluent::Plugin
|
27
|
+
class RecordTransformerFilter < Fluent::Plugin::Filter
|
28
|
+
Fluent::Plugin.register_filter('record_transformer', self)
|
29
|
+
|
30
|
+
desc 'A comma-delimited list of keys to delete.'
|
31
|
+
config_param :remove_keys, :string, default: nil
|
32
|
+
desc 'A comma-delimited list of keys to keep.'
|
33
|
+
config_param :keep_keys, :string, default: nil
|
34
|
+
desc 'Create new Hash to transform incoming data'
|
35
|
+
config_param :renew_record, :bool, default: false
|
36
|
+
desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
|
37
|
+
config_param :renew_time_key, :string, default: nil
|
38
|
+
desc 'When set to true, the full Ruby syntax is enabled in the ${...} expression.'
|
39
|
+
config_param :enable_ruby, :bool, default: false
|
40
|
+
desc 'Use original value type.'
|
41
|
+
config_param :auto_typecast, :bool, default: false # false for lower version compatibility
|
42
|
+
|
43
|
+
def configure(conf)
|
44
|
+
super
|
45
|
+
|
46
|
+
map = {}
|
47
|
+
# <record></record> directive
|
48
|
+
conf.elements.select { |element| element.name == 'record' }.each do |element|
|
49
|
+
element.each_pair do |k, v|
|
50
|
+
element.has_key?(k) # to suppress unread configuration warning
|
51
|
+
map[k] = parse_value(v)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
if @remove_keys
|
56
|
+
@remove_keys = @remove_keys.split(',')
|
57
|
+
end
|
58
|
+
|
59
|
+
if @keep_keys
|
60
|
+
raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
|
61
|
+
@keep_keys = @keep_keys.split(',')
|
62
|
+
end
|
63
|
+
|
64
|
+
placeholder_expander_params = {
|
65
|
+
log: log,
|
66
|
+
auto_typecast: @auto_typecast,
|
67
|
+
}
|
68
|
+
@placeholder_expander =
|
69
|
+
if @enable_ruby
|
70
|
+
# require utilities which would be used in ruby placeholders
|
71
|
+
require 'pathname'
|
72
|
+
require 'uri'
|
73
|
+
require 'cgi'
|
74
|
+
RubyPlaceholderExpander.new(placeholder_expander_params)
|
75
|
+
else
|
76
|
+
PlaceholderExpander.new(placeholder_expander_params)
|
77
|
+
end
|
78
|
+
@map = @placeholder_expander.preprocess_map(map)
|
79
|
+
|
80
|
+
@hostname = Socket.gethostname
|
81
|
+
end
|
82
|
+
|
83
|
+
def filter_stream(tag, es)
|
84
|
+
new_es = Fluent::MultiEventStream.new
|
85
|
+
tag_parts = tag.split('.')
|
86
|
+
tag_prefix = tag_prefix(tag_parts)
|
87
|
+
tag_suffix = tag_suffix(tag_parts)
|
88
|
+
placeholder_values = {
|
89
|
+
'tag' => tag,
|
90
|
+
'tag_parts' => tag_parts,
|
91
|
+
'tag_prefix' => tag_prefix,
|
92
|
+
'tag_suffix' => tag_suffix,
|
93
|
+
'hostname' => @hostname,
|
94
|
+
}
|
95
|
+
last_record = nil
|
96
|
+
es.each do |time, record|
|
97
|
+
last_record = record # for debug log
|
98
|
+
placeholder_values.merge!({
|
99
|
+
'time' => @placeholder_expander.time_value(time),
|
100
|
+
'record' => record,
|
101
|
+
})
|
102
|
+
new_record = reform(record, placeholder_values)
|
103
|
+
if @renew_time_key && new_record.has_key?(@renew_time_key)
|
104
|
+
time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
|
105
|
+
end
|
106
|
+
new_es.add(time, new_record)
|
107
|
+
end
|
108
|
+
new_es
|
109
|
+
rescue => e
|
110
|
+
log.warn "failed to reform records", error: e
|
111
|
+
log.warn_backtrace
|
112
|
+
log.debug "map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def parse_value(value_str)
|
118
|
+
if value_str.start_with?('{', '[')
|
119
|
+
JSON.parse(value_str)
|
120
|
+
else
|
121
|
+
value_str
|
122
|
+
end
|
123
|
+
rescue => e
|
124
|
+
log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", error: e
|
125
|
+
value_str # emit as string
|
126
|
+
end
|
127
|
+
|
128
|
+
def reform(record, placeholder_values)
|
129
|
+
placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
|
130
|
+
|
131
|
+
new_record = @renew_record ? {} : record.dup
|
132
|
+
@keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
|
133
|
+
new_record.merge!(expand_placeholders(@map, placeholders))
|
134
|
+
@remove_keys.each {|k| new_record.delete(k) } if @remove_keys
|
135
|
+
|
136
|
+
new_record
|
137
|
+
end
|
138
|
+
|
139
|
+
def expand_placeholders(value, placeholders)
|
140
|
+
if value.is_a?(String)
|
141
|
+
new_value = @placeholder_expander.expand(value, placeholders)
|
142
|
+
elsif value.is_a?(Hash)
|
143
|
+
new_value = {}
|
144
|
+
value.each_pair do |k, v|
|
145
|
+
new_key = @placeholder_expander.expand(k, placeholders, true)
|
146
|
+
new_value[new_key] = expand_placeholders(v, placeholders)
|
147
|
+
end
|
148
|
+
elsif value.is_a?(Array)
|
149
|
+
new_value = []
|
150
|
+
value.each_with_index do |v, i|
|
151
|
+
new_value[i] = expand_placeholders(v, placeholders)
|
152
|
+
end
|
153
|
+
else
|
154
|
+
new_value = value
|
155
|
+
end
|
156
|
+
new_value
|
157
|
+
end
|
158
|
+
|
159
|
+
def tag_prefix(tag_parts)
|
160
|
+
return [] if tag_parts.empty?
|
161
|
+
tag_prefix = [tag_parts.first]
|
162
|
+
1.upto(tag_parts.size-1).each do |i|
|
163
|
+
tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
|
164
|
+
end
|
165
|
+
tag_prefix
|
166
|
+
end
|
167
|
+
|
168
|
+
def tag_suffix(tag_parts)
|
169
|
+
return [] if tag_parts.empty?
|
170
|
+
rev_tag_parts = tag_parts.reverse
|
171
|
+
rev_tag_suffix = [rev_tag_parts.first]
|
172
|
+
1.upto(tag_parts.size-1).each do |i|
|
173
|
+
rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
|
174
|
+
end
|
175
|
+
rev_tag_suffix.reverse!
|
176
|
+
end
|
177
|
+
|
178
|
+
# THIS CLASS MUST BE THREAD-SAFE
|
179
|
+
class PlaceholderExpander
|
180
|
+
attr_reader :placeholders, :log
|
181
|
+
|
182
|
+
def initialize(params)
|
183
|
+
@log = params[:log]
|
184
|
+
@auto_typecast = params[:auto_typecast]
|
185
|
+
end
|
186
|
+
|
187
|
+
def time_value(time)
|
188
|
+
Time.at(time).to_s
|
189
|
+
end
|
190
|
+
|
191
|
+
def preprocess_map(value, force_stringify = false)
|
192
|
+
value
|
193
|
+
end
|
194
|
+
|
195
|
+
def prepare_placeholders(placeholder_values)
|
196
|
+
placeholders = {}
|
197
|
+
|
198
|
+
placeholder_values.each do |key, value|
|
199
|
+
if value.kind_of?(Array) # tag_parts, etc
|
200
|
+
size = value.size
|
201
|
+
value.each_with_index do |v, idx|
|
202
|
+
placeholders.store("${#{key}[#{idx}]}", v)
|
203
|
+
placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
|
204
|
+
end
|
205
|
+
elsif value.kind_of?(Hash) # record, etc
|
206
|
+
value.each do |k, v|
|
207
|
+
unless placeholder_values.has_key?(k) # prevent overwriting reserved keys such as tag
|
208
|
+
placeholders.store("${#{k}}", v) # foo
|
209
|
+
end
|
210
|
+
placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
|
211
|
+
end
|
212
|
+
else # string, interger, float, and others?
|
213
|
+
placeholders.store("${#{key}}", value)
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
placeholders
|
218
|
+
end
|
219
|
+
|
220
|
+
# Expand string with placeholders
|
221
|
+
#
|
222
|
+
# @param [String] str
|
223
|
+
# @param [Boolean] force_stringify the value must be string, used for hash key
|
224
|
+
def expand(str, placeholders, force_stringify = false)
|
225
|
+
if @auto_typecast and !force_stringify
|
226
|
+
single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
|
227
|
+
if single_placeholder_matched
|
228
|
+
log_if_unknown_placeholder($1, placeholders)
|
229
|
+
return placeholders[single_placeholder_matched[1]]
|
230
|
+
end
|
231
|
+
end
|
232
|
+
str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
|
233
|
+
log_if_unknown_placeholder($1, placeholders)
|
234
|
+
placeholders[$1]
|
235
|
+
}
|
236
|
+
end
|
237
|
+
|
238
|
+
private
|
239
|
+
|
240
|
+
def log_if_unknown_placeholder(placeholder, placeholders)
|
241
|
+
unless placeholders.include?(placeholder)
|
242
|
+
log.warn "unknown placeholder `#{placeholder}` found"
|
243
|
+
end
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
# THIS CLASS MUST BE THREAD-SAFE
|
248
|
+
class RubyPlaceholderExpander
|
249
|
+
attr_reader :log
|
250
|
+
|
251
|
+
def initialize(params)
|
252
|
+
@log = params[:log]
|
253
|
+
@auto_typecast = params[:auto_typecast]
|
254
|
+
@cleanroom_expander = CleanroomExpander.new
|
255
|
+
end
|
256
|
+
|
257
|
+
def time_value(time)
|
258
|
+
Time.at(time)
|
259
|
+
end
|
260
|
+
|
261
|
+
# Preprocess record map to convert into ruby string expansion
|
262
|
+
#
|
263
|
+
# @param [Hash|String|Array] value record map config
|
264
|
+
# @param [Boolean] force_stringify the value must be string, used for hash key
|
265
|
+
def preprocess_map(value, force_stringify = false)
|
266
|
+
new_value = nil
|
267
|
+
if value.is_a?(String)
|
268
|
+
if @auto_typecast and !force_stringify
|
269
|
+
num_placeholders = value.scan('${').size
|
270
|
+
if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
|
271
|
+
new_value = value[2..-2] # ${..} => ..
|
272
|
+
end
|
273
|
+
end
|
274
|
+
unless new_value
|
275
|
+
new_value = "%Q[#{value.gsub('${', '#{')}]" # xx${..}xx => %Q[xx#{..}xx]
|
276
|
+
end
|
277
|
+
elsif value.is_a?(Hash)
|
278
|
+
new_value = {}
|
279
|
+
value.each_pair do |k, v|
|
280
|
+
new_value[preprocess_map(k, true)] = preprocess_map(v)
|
281
|
+
end
|
282
|
+
elsif value.is_a?(Array)
|
283
|
+
new_value = []
|
284
|
+
value.each_with_index do |v, i|
|
285
|
+
new_value[i] = preprocess_map(v)
|
286
|
+
end
|
287
|
+
else
|
288
|
+
new_value = value
|
289
|
+
end
|
290
|
+
new_value
|
291
|
+
end
|
292
|
+
|
293
|
+
def prepare_placeholders(placeholder_values)
|
294
|
+
placeholder_values
|
295
|
+
end
|
296
|
+
|
297
|
+
# Expand string with placeholders
|
298
|
+
#
|
299
|
+
# @param [String] str
|
300
|
+
def expand(str, placeholders, force_stringify = false)
|
301
|
+
@cleanroom_expander.expand(
|
302
|
+
str,
|
303
|
+
placeholders['tag'],
|
304
|
+
placeholders['time'],
|
305
|
+
placeholders['record'],
|
306
|
+
placeholders['tag_parts'],
|
307
|
+
placeholders['tag_prefix'],
|
308
|
+
placeholders['tag_suffix'],
|
309
|
+
placeholders['hostname'],
|
310
|
+
)
|
311
|
+
rescue => e
|
312
|
+
log.warn "failed to expand `#{str}`", error: e
|
313
|
+
log.warn_backtrace
|
314
|
+
nil
|
315
|
+
end
|
316
|
+
|
317
|
+
class CleanroomExpander
|
318
|
+
def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
|
319
|
+
tags = tag_parts # for old version compatibility
|
320
|
+
_ = tags # to suppress "unused variable" warning for tags
|
321
|
+
Thread.current[:record_transformer_record] = record # for old version compatibility
|
322
|
+
instance_eval(__str_to_eval__)
|
323
|
+
end
|
324
|
+
|
325
|
+
# for old version compatibility
|
326
|
+
def method_missing(name)
|
327
|
+
key = name.to_s
|
328
|
+
record = Thread.current[:record_transformer_record]
|
329
|
+
if record.has_key?(key)
|
330
|
+
record[key]
|
331
|
+
else
|
332
|
+
raise NameError, "undefined local variable or method `#{key}'"
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
(Object.instance_methods).each do |m|
|
337
|
+
undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
|
338
|
+
end
|
339
|
+
end
|
340
|
+
end
|
341
|
+
end
|
342
|
+
end
|