fluentd 1.14.4-x64-mingw-ucrt
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.deepsource.toml +13 -0
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +70 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
- data/.github/ISSUE_TEMPLATE.md +17 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +14 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux-test.yaml +36 -0
- data/.github/workflows/macos-test.yaml +30 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.github/workflows/windows-test.yaml +46 -0
- data/.gitignore +30 -0
- data/.gitlab-ci.yml +103 -0
- data/ADOPTERS.md +5 -0
- data/AUTHORS +2 -0
- data/CHANGELOG.md +2409 -0
- data/CONTRIBUTING.md +45 -0
- data/GOVERNANCE.md +55 -0
- data/Gemfile +9 -0
- data/GithubWorkflow.md +78 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +11 -0
- data/README.md +97 -0
- data/Rakefile +79 -0
- data/SECURITY.md +18 -0
- data/bin/fluent-binlog-reader +7 -0
- data/bin/fluent-ca-generate +6 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-cat +5 -0
- data/bin/fluent-ctl +7 -0
- data/bin/fluent-debug +5 -0
- data/bin/fluent-gem +9 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/bin/fluentd +15 -0
- data/code-of-conduct.md +3 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/example/copy_roundrobin.conf +39 -0
- data/example/counter.conf +18 -0
- data/example/filter_stdout.conf +22 -0
- data/example/in_forward.conf +14 -0
- data/example/in_forward_client.conf +37 -0
- data/example/in_forward_shared_key.conf +15 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_users.conf +24 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/in_http.conf +16 -0
- data/example/in_out_forward.conf +17 -0
- data/example/in_sample_blocks.conf +17 -0
- data/example/in_sample_with_compression.conf +23 -0
- data/example/in_syslog.conf +15 -0
- data/example/in_tail.conf +14 -0
- data/example/in_tcp.conf +13 -0
- data/example/in_udp.conf +13 -0
- data/example/logevents.conf +25 -0
- data/example/multi_filters.conf +61 -0
- data/example/out_copy.conf +20 -0
- data/example/out_exec_filter.conf +42 -0
- data/example/out_file.conf +13 -0
- data/example/out_forward.conf +35 -0
- data/example/out_forward_buf_file.conf +23 -0
- data/example/out_forward_client.conf +109 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_sd.conf +17 -0
- data/example/out_forward_shared_key.conf +36 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/out_forward_users.conf +65 -0
- data/example/out_null.conf +36 -0
- data/example/sd.yaml +8 -0
- data/example/secondary_file.conf +42 -0
- data/example/suppress_config_dump.conf +7 -0
- data/example/v0_12_filter.conf +78 -0
- data/example/v1_literal_example.conf +36 -0
- data/example/worker_section.conf +36 -0
- data/fluent.conf +139 -0
- data/fluentd.gemspec +55 -0
- data/lib/fluent/agent.rb +168 -0
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/clock.rb +66 -0
- data/lib/fluent/command/binlog_reader.rb +244 -0
- data/lib/fluent/command/bundler_injection.rb +45 -0
- data/lib/fluent/command/ca_generate.rb +184 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/cat.rb +365 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/debug.rb +103 -0
- data/lib/fluent/command/fluentd.rb +374 -0
- data/lib/fluent/command/plugin_config_formatter.rb +308 -0
- data/lib/fluent/command/plugin_generator.rb +365 -0
- data/lib/fluent/compat/call_super_mixin.rb +76 -0
- data/lib/fluent/compat/detach_process_mixin.rb +33 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +68 -0
- data/lib/fluent/compat/formatter.rb +111 -0
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +49 -0
- data/lib/fluent/compat/output.rb +721 -0
- data/lib/fluent/compat/output_chain.rb +60 -0
- data/lib/fluent/compat/parser.rb +310 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/propagate_default.rb +62 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/compat/structured_format_mixin.rb +26 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/basic_parser.rb +123 -0
- data/lib/fluent/config/configure_proxy.rb +424 -0
- data/lib/fluent/config/dsl.rb +152 -0
- data/lib/fluent/config/element.rb +265 -0
- data/lib/fluent/config/error.rb +32 -0
- data/lib/fluent/config/literal_parser.rb +286 -0
- data/lib/fluent/config/parser.rb +107 -0
- data/lib/fluent/config/section.rb +272 -0
- data/lib/fluent/config/types.rb +249 -0
- data/lib/fluent/config/v1_parser.rb +192 -0
- data/lib/fluent/config.rb +76 -0
- data/lib/fluent/configurable.rb +201 -0
- data/lib/fluent/counter/base_socket.rb +44 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/daemon.rb +15 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/engine.rb +253 -0
- data/lib/fluent/env.rb +40 -0
- data/lib/fluent/error.rb +34 -0
- data/lib/fluent/event.rb +326 -0
- data/lib/fluent/event_router.rb +297 -0
- data/lib/fluent/ext_monitor_require.rb +28 -0
- data/lib/fluent/filter.rb +21 -0
- data/lib/fluent/fluent_log_event_router.rb +141 -0
- data/lib/fluent/formatter.rb +23 -0
- data/lib/fluent/input.rb +21 -0
- data/lib/fluent/label.rb +46 -0
- data/lib/fluent/load.rb +34 -0
- data/lib/fluent/log.rb +713 -0
- data/lib/fluent/match.rb +187 -0
- data/lib/fluent/mixin.rb +31 -0
- data/lib/fluent/msgpack_factory.rb +106 -0
- data/lib/fluent/oj_options.rb +62 -0
- data/lib/fluent/output.rb +29 -0
- data/lib/fluent/output_chain.rb +23 -0
- data/lib/fluent/parser.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +104 -0
- data/lib/fluent/plugin/base.rb +197 -0
- data/lib/fluent/plugin/buf_file.rb +213 -0
- data/lib/fluent/plugin/buf_file_single.rb +225 -0
- data/lib/fluent/plugin/buf_memory.rb +34 -0
- data/lib/fluent/plugin/buffer/chunk.rb +240 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +413 -0
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +311 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +91 -0
- data/lib/fluent/plugin/buffer.rb +918 -0
- data/lib/fluent/plugin/compressable.rb +96 -0
- data/lib/fluent/plugin/exec_util.rb +22 -0
- data/lib/fluent/plugin/file_util.rb +22 -0
- data/lib/fluent/plugin/file_wrapper.rb +187 -0
- data/lib/fluent/plugin/filter.rb +127 -0
- data/lib/fluent/plugin/filter_grep.rb +189 -0
- data/lib/fluent/plugin/filter_parser.rb +130 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +324 -0
- data/lib/fluent/plugin/filter_stdout.rb +53 -0
- data/lib/fluent/plugin/formatter.rb +75 -0
- data/lib/fluent/plugin/formatter_csv.rb +78 -0
- data/lib/fluent/plugin/formatter_hash.rb +35 -0
- data/lib/fluent/plugin/formatter_json.rb +59 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +44 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +33 -0
- data/lib/fluent/plugin/formatter_out_file.rb +53 -0
- data/lib/fluent/plugin/formatter_single_value.rb +36 -0
- data/lib/fluent/plugin/formatter_stdout.rb +76 -0
- data/lib/fluent/plugin/formatter_tsv.rb +40 -0
- data/lib/fluent/plugin/in_debug_agent.rb +71 -0
- data/lib/fluent/plugin/in_dummy.rb +18 -0
- data/lib/fluent/plugin/in_exec.rb +110 -0
- data/lib/fluent/plugin/in_forward.rb +473 -0
- data/lib/fluent/plugin/in_gc_stat.rb +72 -0
- data/lib/fluent/plugin/in_http.rb +667 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +412 -0
- data/lib/fluent/plugin/in_object_space.rb +93 -0
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_syslog.rb +276 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +269 -0
- data/lib/fluent/plugin/in_tail.rb +1228 -0
- data/lib/fluent/plugin/in_tcp.rb +181 -0
- data/lib/fluent/plugin/in_udp.rb +92 -0
- data/lib/fluent/plugin/in_unix.rb +195 -0
- data/lib/fluent/plugin/input.rb +75 -0
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +195 -0
- data/lib/fluent/plugin/out_copy.rb +120 -0
- data/lib/fluent/plugin/out_exec.rb +105 -0
- data/lib/fluent/plugin/out_exec_filter.rb +319 -0
- data/lib/fluent/plugin/out_file.rb +334 -0
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +125 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +114 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +140 -0
- data/lib/fluent/plugin/out_forward.rb +826 -0
- data/lib/fluent/plugin/out_http.rb +275 -0
- data/lib/fluent/plugin/out_null.rb +74 -0
- data/lib/fluent/plugin/out_relabel.rb +32 -0
- data/lib/fluent/plugin/out_roundrobin.rb +84 -0
- data/lib/fluent/plugin/out_secondary_file.rb +131 -0
- data/lib/fluent/plugin/out_stdout.rb +74 -0
- data/lib/fluent/plugin/out_stream.rb +130 -0
- data/lib/fluent/plugin/output.rb +1556 -0
- data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
- data/lib/fluent/plugin/parser.rb +275 -0
- data/lib/fluent/plugin/parser_apache.rb +28 -0
- data/lib/fluent/plugin/parser_apache2.rb +88 -0
- data/lib/fluent/plugin/parser_apache_error.rb +26 -0
- data/lib/fluent/plugin/parser_csv.rb +114 -0
- data/lib/fluent/plugin/parser_json.rb +96 -0
- data/lib/fluent/plugin/parser_ltsv.rb +51 -0
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_multiline.rb +152 -0
- data/lib/fluent/plugin/parser_nginx.rb +28 -0
- data/lib/fluent/plugin/parser_none.rb +36 -0
- data/lib/fluent/plugin/parser_regexp.rb +68 -0
- data/lib/fluent/plugin/parser_syslog.rb +496 -0
- data/lib/fluent/plugin/parser_tsv.rb +42 -0
- data/lib/fluent/plugin/sd_file.rb +156 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin/sd_static.rb +58 -0
- data/lib/fluent/plugin/service_discovery.rb +65 -0
- data/lib/fluent/plugin/socket_util.rb +22 -0
- data/lib/fluent/plugin/storage.rb +84 -0
- data/lib/fluent/plugin/storage_local.rb +162 -0
- data/lib/fluent/plugin/string_util.rb +22 -0
- data/lib/fluent/plugin.rb +206 -0
- data/lib/fluent/plugin_helper/cert_option.rb +191 -0
- data/lib/fluent/plugin_helper/child_process.rb +366 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +343 -0
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +100 -0
- data/lib/fluent/plugin_helper/event_loop.rb +170 -0
- data/lib/fluent/plugin_helper/extract.rb +104 -0
- data/lib/fluent/plugin_helper/formatter.rb +147 -0
- data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +92 -0
- data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
- data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
- data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
- data/lib/fluent/plugin_helper/http_server/server.rb +93 -0
- data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
- data/lib/fluent/plugin_helper/http_server.rb +135 -0
- data/lib/fluent/plugin_helper/inject.rb +154 -0
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/parser.rb +147 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +207 -0
- data/lib/fluent/plugin_helper/retry_state.rb +209 -0
- data/lib/fluent/plugin_helper/server.rb +801 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +146 -0
- data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +43 -0
- data/lib/fluent/plugin_helper/service_discovery.rb +125 -0
- data/lib/fluent/plugin_helper/socket.rb +277 -0
- data/lib/fluent/plugin_helper/socket_option.rb +98 -0
- data/lib/fluent/plugin_helper/storage.rb +349 -0
- data/lib/fluent/plugin_helper/thread.rb +180 -0
- data/lib/fluent/plugin_helper/timer.rb +92 -0
- data/lib/fluent/plugin_helper.rb +75 -0
- data/lib/fluent/plugin_id.rb +93 -0
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/registry.rb +116 -0
- data/lib/fluent/root_agent.rb +372 -0
- data/lib/fluent/rpc.rb +94 -0
- data/lib/fluent/static_config_analysis.rb +194 -0
- data/lib/fluent/supervisor.rb +1054 -0
- data/lib/fluent/system_config.rb +187 -0
- data/lib/fluent/test/base.rb +78 -0
- data/lib/fluent/test/driver/base.rb +225 -0
- data/lib/fluent/test/driver/base_owned.rb +83 -0
- data/lib/fluent/test/driver/base_owner.rb +135 -0
- data/lib/fluent/test/driver/event_feeder.rb +98 -0
- data/lib/fluent/test/driver/filter.rb +57 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/input.rb +31 -0
- data/lib/fluent/test/driver/multi_output.rb +53 -0
- data/lib/fluent/test/driver/output.rb +102 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/driver/storage.rb +30 -0
- data/lib/fluent/test/driver/test_event_router.rb +45 -0
- data/lib/fluent/test/filter_test.rb +77 -0
- data/lib/fluent/test/formatter_test.rb +65 -0
- data/lib/fluent/test/helpers.rb +134 -0
- data/lib/fluent/test/input_test.rb +174 -0
- data/lib/fluent/test/log.rb +79 -0
- data/lib/fluent/test/output_test.rb +156 -0
- data/lib/fluent/test/parser_test.rb +70 -0
- data/lib/fluent/test/startup_shutdown.rb +46 -0
- data/lib/fluent/test.rb +58 -0
- data/lib/fluent/time.rb +512 -0
- data/lib/fluent/timezone.rb +171 -0
- data/lib/fluent/tls.rb +81 -0
- data/lib/fluent/unique_id.rb +39 -0
- data/lib/fluent/variable_store.rb +40 -0
- data/lib/fluent/version.rb +21 -0
- data/lib/fluent/winsvc.rb +103 -0
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md-table.erb +10 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +362 -0
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_cat.rb +128 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_fluentd.rb +1106 -0
- data/test/command/test_plugin_config_formatter.rb +398 -0
- data/test/command/test_plugin_generator.rb +109 -0
- data/test/compat/test_calls_super.rb +166 -0
- data/test/compat/test_parser.rb +92 -0
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +551 -0
- data/test/config/test_configurable.rb +1784 -0
- data/test/config/test_configure_proxy.rb +604 -0
- data/test/config/test_dsl.rb +415 -0
- data/test/config/test_element.rb +518 -0
- data/test/config/test_literal_parser.rb +309 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_section.rb +191 -0
- data/test/config/test_system_config.rb +199 -0
- data/test/config/test_types.rb +408 -0
- data/test/counter/test_client.rb +563 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/helper.rb +155 -0
- data/test/helpers/fuzzy_assert.rb +89 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/sd_file/config +11 -0
- data/test/plugin/data/sd_file/config.json +17 -0
- data/test/plugin/data/sd_file/config.yaml +11 -0
- data/test/plugin/data/sd_file/config.yml +11 -0
- data/test/plugin/data/sd_file/invalid_config.yml +7 -0
- data/test/plugin/in_tail/test_fifo.rb +121 -0
- data/test/plugin/in_tail/test_io_handler.rb +140 -0
- data/test/plugin/in_tail/test_position_file.rb +379 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +112 -0
- data/test/plugin/out_forward/test_load_balancer.rb +106 -0
- data/test/plugin/out_forward/test_socket_cache.rb +149 -0
- data/test/plugin/test_bare_output.rb +131 -0
- data/test/plugin/test_base.rb +115 -0
- data/test/plugin/test_buf_file.rb +1275 -0
- data/test/plugin/test_buf_file_single.rb +833 -0
- data/test/plugin/test_buf_memory.rb +42 -0
- data/test/plugin/test_buffer.rb +1383 -0
- data/test/plugin/test_buffer_chunk.rb +198 -0
- data/test/plugin/test_buffer_file_chunk.rb +871 -0
- data/test/plugin/test_buffer_file_single_chunk.rb +611 -0
- data/test/plugin/test_buffer_memory_chunk.rb +339 -0
- data/test/plugin/test_compressable.rb +87 -0
- data/test/plugin/test_file_util.rb +96 -0
- data/test/plugin/test_file_wrapper.rb +126 -0
- data/test/plugin/test_filter.rb +368 -0
- data/test/plugin/test_filter_grep.rb +697 -0
- data/test/plugin/test_filter_parser.rb +731 -0
- data/test/plugin/test_filter_record_transformer.rb +577 -0
- data/test/plugin/test_filter_stdout.rb +207 -0
- data/test/plugin/test_formatter_csv.rb +136 -0
- data/test/plugin/test_formatter_hash.rb +38 -0
- data/test/plugin/test_formatter_json.rb +61 -0
- data/test/plugin/test_formatter_ltsv.rb +70 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +116 -0
- data/test/plugin/test_formatter_single_value.rb +44 -0
- data/test/plugin/test_formatter_tsv.rb +76 -0
- data/test/plugin/test_in_debug_agent.rb +49 -0
- data/test/plugin/test_in_exec.rb +261 -0
- data/test/plugin/test_in_forward.rb +1180 -0
- data/test/plugin/test_in_gc_stat.rb +62 -0
- data/test/plugin/test_in_http.rb +1080 -0
- data/test/plugin/test_in_monitor_agent.rb +923 -0
- data/test/plugin/test_in_object_space.rb +60 -0
- data/test/plugin/test_in_sample.rb +190 -0
- data/test/plugin/test_in_syslog.rb +505 -0
- data/test/plugin/test_in_tail.rb +2363 -0
- data/test/plugin/test_in_tcp.rb +243 -0
- data/test/plugin/test_in_udp.rb +268 -0
- data/test/plugin/test_in_unix.rb +181 -0
- data/test/plugin/test_input.rb +137 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +204 -0
- data/test/plugin/test_out_copy.rb +308 -0
- data/test/plugin/test_out_exec.rb +312 -0
- data/test/plugin/test_out_exec_filter.rb +606 -0
- data/test/plugin/test_out_file.rb +1037 -0
- data/test/plugin/test_out_forward.rb +1348 -0
- data/test/plugin/test_out_http.rb +428 -0
- data/test/plugin/test_out_null.rb +105 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +146 -0
- data/test/plugin/test_out_secondary_file.rb +458 -0
- data/test/plugin/test_out_stdout.rb +205 -0
- data/test/plugin/test_out_stream.rb +103 -0
- data/test/plugin/test_output.rb +1065 -0
- data/test/plugin/test_output_as_buffered.rb +2024 -0
- data/test/plugin/test_output_as_buffered_backup.rb +363 -0
- data/test/plugin/test_output_as_buffered_compress.rb +165 -0
- data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
- data/test/plugin/test_output_as_buffered_retries.rb +919 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +882 -0
- data/test/plugin/test_output_as_standard.rb +374 -0
- data/test/plugin/test_owned_by.rb +35 -0
- data/test/plugin/test_parser.rb +399 -0
- data/test/plugin/test_parser_apache.rb +42 -0
- data/test/plugin/test_parser_apache2.rb +47 -0
- data/test/plugin/test_parser_apache_error.rb +45 -0
- data/test/plugin/test_parser_csv.rb +200 -0
- data/test/plugin/test_parser_json.rb +138 -0
- data/test/plugin/test_parser_labeled_tsv.rb +160 -0
- data/test/plugin/test_parser_multiline.rb +111 -0
- data/test/plugin/test_parser_nginx.rb +88 -0
- data/test/plugin/test_parser_none.rb +52 -0
- data/test/plugin/test_parser_regexp.rb +289 -0
- data/test/plugin/test_parser_syslog.rb +650 -0
- data/test/plugin/test_parser_tsv.rb +122 -0
- data/test/plugin/test_sd_file.rb +228 -0
- data/test/plugin/test_sd_srv.rb +230 -0
- data/test/plugin/test_storage.rb +167 -0
- data/test/plugin/test_storage_local.rb +335 -0
- data/test/plugin/test_string_util.rb +26 -0
- data/test/plugin_helper/data/cert/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
- data/test/plugin_helper/data/cert/cert.pem +19 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +125 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
- data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
- data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
- data/test/plugin_helper/http_server/test_app.rb +65 -0
- data/test/plugin_helper/http_server/test_route.rb +32 -0
- data/test/plugin_helper/service_discovery/test_manager.rb +93 -0
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +21 -0
- data/test/plugin_helper/test_cert_option.rb +25 -0
- data/test/plugin_helper/test_child_process.rb +840 -0
- data/test/plugin_helper/test_compat_parameters.rb +358 -0
- data/test/plugin_helper/test_event_emitter.rb +80 -0
- data/test/plugin_helper/test_event_loop.rb +52 -0
- data/test/plugin_helper/test_extract.rb +194 -0
- data/test/plugin_helper/test_formatter.rb +255 -0
- data/test/plugin_helper/test_http_server_helper.rb +372 -0
- data/test/plugin_helper/test_inject.rb +561 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_parser.rb +264 -0
- data/test/plugin_helper/test_record_accessor.rb +238 -0
- data/test/plugin_helper/test_retry_state.rb +442 -0
- data/test/plugin_helper/test_server.rb +1823 -0
- data/test/plugin_helper/test_service_discovery.rb +165 -0
- data/test/plugin_helper/test_socket.rb +146 -0
- data/test/plugin_helper/test_storage.rb +542 -0
- data/test/plugin_helper/test_thread.rb +164 -0
- data/test/plugin_helper/test_timer.rb +130 -0
- data/test/scripts/exec_script.rb +32 -0
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
- data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
- data/test/scripts/fluent/plugin/out_test.rb +81 -0
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/scripts/fluent/plugin/parser_known.rb +4 -0
- data/test/test_capability.rb +74 -0
- data/test/test_clock.rb +164 -0
- data/test/test_config.rb +202 -0
- data/test/test_configdsl.rb +148 -0
- data/test/test_daemonizer.rb +91 -0
- data/test/test_engine.rb +203 -0
- data/test/test_event.rb +531 -0
- data/test/test_event_router.rb +331 -0
- data/test/test_event_time.rb +199 -0
- data/test/test_filter.rb +121 -0
- data/test/test_fluent_log_event_router.rb +99 -0
- data/test/test_formatter.rb +366 -0
- data/test/test_input.rb +31 -0
- data/test/test_log.rb +994 -0
- data/test/test_logger_initializer.rb +46 -0
- data/test/test_match.rb +148 -0
- data/test/test_mixin.rb +351 -0
- data/test/test_msgpack_factory.rb +18 -0
- data/test/test_oj_options.rb +55 -0
- data/test/test_output.rb +278 -0
- data/test/test_plugin.rb +251 -0
- data/test/test_plugin_classes.rb +370 -0
- data/test/test_plugin_helper.rb +81 -0
- data/test/test_plugin_id.rb +119 -0
- data/test/test_process.rb +14 -0
- data/test/test_root_agent.rb +951 -0
- data/test/test_static_config_analysis.rb +177 -0
- data/test/test_supervisor.rb +601 -0
- data/test/test_test_drivers.rb +136 -0
- data/test/test_time_formatter.rb +301 -0
- data/test/test_time_parser.rb +362 -0
- data/test/test_tls.rb +65 -0
- data/test/test_unique_id.rb +47 -0
- data/test/test_variable_store.rb +65 -0
- metadata +1261 -0
@@ -0,0 +1,42 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module Plugin
|
19
|
+
module OwnedByMixin
|
20
|
+
def owner=(plugin)
|
21
|
+
@_owner = plugin
|
22
|
+
|
23
|
+
@_plugin_id = plugin.plugin_id
|
24
|
+
@_plugin_id_configured = plugin.plugin_id_configured?
|
25
|
+
|
26
|
+
@log = plugin.log
|
27
|
+
end
|
28
|
+
|
29
|
+
def owner
|
30
|
+
if instance_variable_defined?("@_owner")
|
31
|
+
@_owner
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def log
|
36
|
+
if instance_variable_defined?("@log")
|
37
|
+
@log
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,275 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/base'
|
18
|
+
require 'fluent/plugin/owned_by_mixin'
|
19
|
+
|
20
|
+
require 'fluent/error'
|
21
|
+
require 'fluent/mixin' # for TypeConverter
|
22
|
+
require 'fluent/time'
|
23
|
+
require 'fluent/plugin/string_util'
|
24
|
+
|
25
|
+
require 'serverengine/blocking_flag'
|
26
|
+
|
27
|
+
module Fluent
|
28
|
+
module Plugin
|
29
|
+
class Parser < Base
|
30
|
+
class TimeoutChecker
|
31
|
+
# This implementation now uses mutex because parser is typically used in input.
|
32
|
+
# If this has a performance issue under high concurreny, use concurrent-ruby's map instead.
|
33
|
+
def initialize(timeout)
|
34
|
+
@map = {}
|
35
|
+
@flag = ServerEngine::BlockingFlag.new
|
36
|
+
@mutex = Mutex.new
|
37
|
+
@timeout = timeout
|
38
|
+
@timeout_checker = nil
|
39
|
+
end
|
40
|
+
|
41
|
+
def start
|
42
|
+
@thread = ::Thread.new {
|
43
|
+
until @flag.wait_for_set(0.5)
|
44
|
+
now = Time.now
|
45
|
+
@mutex.synchronize {
|
46
|
+
@map.keys.each { |th|
|
47
|
+
time = @map[th]
|
48
|
+
if now - time > @timeout
|
49
|
+
th.raise UncatchableError, "parsing timed out"
|
50
|
+
@map.delete(th)
|
51
|
+
end
|
52
|
+
}
|
53
|
+
}
|
54
|
+
end
|
55
|
+
}
|
56
|
+
end
|
57
|
+
|
58
|
+
def stop
|
59
|
+
@flag.set!
|
60
|
+
@thread.join
|
61
|
+
end
|
62
|
+
|
63
|
+
def execute
|
64
|
+
th = Thread.current
|
65
|
+
@mutex.synchronize { @map[th] = Time.now }
|
66
|
+
yield
|
67
|
+
ensure
|
68
|
+
# Need clean up here because if next event is delayed, incorrect exception will be raised in normal flow.
|
69
|
+
@mutex.synchronize { @map.delete(th) }
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
include OwnedByMixin
|
74
|
+
include TimeMixin::Parser
|
75
|
+
|
76
|
+
class ParserError < StandardError; end
|
77
|
+
|
78
|
+
configured_in :parse
|
79
|
+
|
80
|
+
### types can be specified as string-based hash style
|
81
|
+
# field1:type, field2:type, field3:type:option, field4:type:option
|
82
|
+
### or, JSON format
|
83
|
+
# {"field1":"type", "field2":"type", "field3":"type:option", "field4":"type:option"}
|
84
|
+
config_param :types, :hash, value_type: :string, default: nil
|
85
|
+
|
86
|
+
# available options are:
|
87
|
+
# array: (1st) delimiter
|
88
|
+
# time : type[, format, timezone] -> type should be a valid "time_type"(string/unixtime/float)
|
89
|
+
# : format[, timezone]
|
90
|
+
|
91
|
+
config_param :time_key, :string, default: nil
|
92
|
+
config_param :null_value_pattern, :string, default: nil
|
93
|
+
config_param :null_empty_string, :bool, default: false
|
94
|
+
config_param :estimate_current_event, :bool, default: true
|
95
|
+
config_param :keep_time_key, :bool, default: false
|
96
|
+
config_param :timeout, :time, default: nil
|
97
|
+
|
98
|
+
AVAILABLE_PARSER_VALUE_TYPES = ['string', 'integer', 'float', 'bool', 'time', 'array']
|
99
|
+
|
100
|
+
# for tests
|
101
|
+
attr_reader :type_converters
|
102
|
+
|
103
|
+
PARSER_TYPES = [:text_per_line, :text, :binary]
|
104
|
+
def parser_type
|
105
|
+
:text_per_line
|
106
|
+
end
|
107
|
+
|
108
|
+
def initialize
|
109
|
+
super
|
110
|
+
|
111
|
+
@timeout_checker = nil
|
112
|
+
end
|
113
|
+
|
114
|
+
def configure(conf)
|
115
|
+
super
|
116
|
+
|
117
|
+
@time_parser = time_parser_create
|
118
|
+
@null_value_regexp = @null_value_pattern && Regexp.new(@null_value_pattern)
|
119
|
+
@type_converters = build_type_converters(@types)
|
120
|
+
@execute_convert_values = @type_converters || @null_value_regexp || @null_empty_string
|
121
|
+
@timeout_checker = if @timeout
|
122
|
+
class << self
|
123
|
+
alias_method :parse_orig, :parse
|
124
|
+
alias_method :parse, :parse_with_timeout
|
125
|
+
end
|
126
|
+
TimeoutChecker.new(@timeout)
|
127
|
+
else
|
128
|
+
nil
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def start
|
133
|
+
super
|
134
|
+
|
135
|
+
@timeout_checker.start if @timeout_checker
|
136
|
+
end
|
137
|
+
|
138
|
+
def stop
|
139
|
+
super
|
140
|
+
|
141
|
+
@timeout_checker.stop if @timeout_checker
|
142
|
+
end
|
143
|
+
|
144
|
+
def parse(text, &block)
|
145
|
+
raise NotImplementedError, "Implement this method in child class"
|
146
|
+
end
|
147
|
+
|
148
|
+
def parse_with_timeout(text, &block)
|
149
|
+
@timeout_checker.execute {
|
150
|
+
parse_orig(text, &block)
|
151
|
+
}
|
152
|
+
rescue UncatchableError
|
153
|
+
log.warn "parsing timed out with #{self.class}: text = #{text}"
|
154
|
+
# Return nil instead of raising error. in_tail or other plugin can emit broken line.
|
155
|
+
yield nil, nil
|
156
|
+
end
|
157
|
+
|
158
|
+
def call(*a, &b)
|
159
|
+
# Keep backward compatibility for existing plugins
|
160
|
+
# TODO: warn when deprecated
|
161
|
+
parse(*a, &b)
|
162
|
+
end
|
163
|
+
|
164
|
+
def implement?(feature)
|
165
|
+
methods_of_plugin = self.class.instance_methods(false)
|
166
|
+
case feature
|
167
|
+
when :parse_io then methods_of_plugin.include?(:parse_io)
|
168
|
+
when :parse_partial_data then methods_of_plugin.include?(:parse_partial_data)
|
169
|
+
else
|
170
|
+
raise ArgumentError, "Unknown feature for parser plugin: #{feature}"
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
def parse_io(io, &block)
|
175
|
+
raise NotImplementedError, "Optional API #parse_io is not implemented"
|
176
|
+
end
|
177
|
+
|
178
|
+
def parse_partial_data(data, &block)
|
179
|
+
raise NotImplementedError, "Optional API #parse_partial_data is not implemented"
|
180
|
+
end
|
181
|
+
|
182
|
+
def parse_time(record)
|
183
|
+
if @time_key && record.respond_to?(:has_key?) && record.has_key?(@time_key)
|
184
|
+
src = if @keep_time_key
|
185
|
+
record[@time_key]
|
186
|
+
else
|
187
|
+
record.delete(@time_key)
|
188
|
+
end
|
189
|
+
@time_parser.parse(src)
|
190
|
+
elsif @estimate_current_event
|
191
|
+
Fluent::EventTime.now
|
192
|
+
else
|
193
|
+
nil
|
194
|
+
end
|
195
|
+
rescue Fluent::TimeParser::TimeParseError => e
|
196
|
+
raise ParserError, e.message
|
197
|
+
end
|
198
|
+
|
199
|
+
# def parse(text, &block)
|
200
|
+
# time, record = convert_values(time, record)
|
201
|
+
# yield time, record
|
202
|
+
# end
|
203
|
+
def convert_values(time, record)
|
204
|
+
return time, record unless @execute_convert_values
|
205
|
+
|
206
|
+
record.each_key do |key|
|
207
|
+
value = record[key]
|
208
|
+
next unless value # nil/null value is always left as-is.
|
209
|
+
|
210
|
+
if value.is_a?(String) && string_like_null(value)
|
211
|
+
record[key] = nil
|
212
|
+
next
|
213
|
+
end
|
214
|
+
|
215
|
+
if @type_converters && @type_converters.has_key?(key)
|
216
|
+
record[key] = @type_converters[key].call(value)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
return time, record
|
221
|
+
end
|
222
|
+
|
223
|
+
def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_regexp)
|
224
|
+
null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
|
225
|
+
end
|
226
|
+
|
227
|
+
TRUTHY_VALUES = ['true', 'yes', '1']
|
228
|
+
|
229
|
+
def build_type_converters(types)
|
230
|
+
return nil unless types
|
231
|
+
|
232
|
+
converters = {}
|
233
|
+
|
234
|
+
types.each_pair do |field_name, type_definition|
|
235
|
+
type, option = type_definition.split(":", 2)
|
236
|
+
unless AVAILABLE_PARSER_VALUE_TYPES.include?(type)
|
237
|
+
raise Fluent::ConfigError, "unknown value conversion for key:'#{field_name}', type:'#{type}'"
|
238
|
+
end
|
239
|
+
|
240
|
+
conv = case type
|
241
|
+
when 'string' then ->(v){ v.to_s }
|
242
|
+
when 'integer' then ->(v){ v.to_i rescue v.to_s.to_i }
|
243
|
+
when 'float' then ->(v){ v.to_f rescue v.to_s.to_f }
|
244
|
+
when 'bool' then ->(v){ TRUTHY_VALUES.include?(v.to_s.downcase) }
|
245
|
+
when 'time'
|
246
|
+
# comma-separated: time:[timezone:]time_format
|
247
|
+
# time_format is unixtime/float/string-time-format
|
248
|
+
timep = if option
|
249
|
+
time_type = 'string' # estimate
|
250
|
+
timezone, time_format = option.split(':', 2)
|
251
|
+
unless Fluent::Timezone.validate(timezone)
|
252
|
+
timezone, time_format = nil, option
|
253
|
+
end
|
254
|
+
if Fluent::TimeMixin::TIME_TYPES.include?(time_format)
|
255
|
+
time_type, time_format = time_format, nil # unixtime/float
|
256
|
+
end
|
257
|
+
time_parser_create(type: time_type.to_sym, format: time_format, timezone: timezone)
|
258
|
+
else
|
259
|
+
time_parser_create(type: :string, format: nil, timezone: nil)
|
260
|
+
end
|
261
|
+
->(v){ timep.parse(v) rescue nil }
|
262
|
+
when 'array'
|
263
|
+
delimiter = option ? option.to_s : ','
|
264
|
+
->(v){ string_safe_encoding(v.to_s){|s| s.split(delimiter) } }
|
265
|
+
else
|
266
|
+
raise "BUG: unknown type even after check: #{type}"
|
267
|
+
end
|
268
|
+
converters[field_name] = conv
|
269
|
+
end
|
270
|
+
|
271
|
+
converters
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/parser_regexp'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module Plugin
|
21
|
+
class ApacheParser < RegexpParser
|
22
|
+
Plugin.register_parser("apache", self)
|
23
|
+
|
24
|
+
config_set_default :expression, /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
|
25
|
+
config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/parser'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module Plugin
|
21
|
+
class Apache2Parser < Parser
|
22
|
+
Plugin.register_parser('apache2', self)
|
23
|
+
|
24
|
+
REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\")*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\")*)" "(?<agent>(?:[^\"]|\\")*)")?$/
|
25
|
+
TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
|
26
|
+
|
27
|
+
def initialize
|
28
|
+
super
|
29
|
+
@mutex = Mutex.new
|
30
|
+
end
|
31
|
+
|
32
|
+
def configure(conf)
|
33
|
+
super
|
34
|
+
@time_parser = time_parser_create(format: TIME_FORMAT)
|
35
|
+
end
|
36
|
+
|
37
|
+
def patterns
|
38
|
+
{'format' => REGEXP, 'time_format' => TIME_FORMAT}
|
39
|
+
end
|
40
|
+
|
41
|
+
def parse(text)
|
42
|
+
m = REGEXP.match(text)
|
43
|
+
unless m
|
44
|
+
yield nil, nil
|
45
|
+
return
|
46
|
+
end
|
47
|
+
|
48
|
+
host = m['host']
|
49
|
+
host = (host == '-') ? nil : host
|
50
|
+
|
51
|
+
user = m['user']
|
52
|
+
user = (user == '-') ? nil : user
|
53
|
+
|
54
|
+
time = m['time']
|
55
|
+
time = @mutex.synchronize { @time_parser.parse(time) }
|
56
|
+
|
57
|
+
method = m['method']
|
58
|
+
path = m['path']
|
59
|
+
|
60
|
+
code = m['code'].to_i
|
61
|
+
code = nil if code == 0
|
62
|
+
|
63
|
+
size = m['size']
|
64
|
+
size = (size == '-') ? nil : size.to_i
|
65
|
+
|
66
|
+
referer = m['referer']
|
67
|
+
referer = (referer == '-') ? nil : referer
|
68
|
+
|
69
|
+
agent = m['agent']
|
70
|
+
agent = (agent == '-') ? nil : agent
|
71
|
+
|
72
|
+
record = {
|
73
|
+
"host" => host,
|
74
|
+
"user" => user,
|
75
|
+
"method" => method,
|
76
|
+
"path" => path,
|
77
|
+
"code" => code,
|
78
|
+
"size" => size,
|
79
|
+
"referer" => referer,
|
80
|
+
"agent" => agent,
|
81
|
+
}
|
82
|
+
record["time"] = m['time'] if @keep_time_key
|
83
|
+
|
84
|
+
yield time, record
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/parser_regexp'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module Plugin
|
21
|
+
class ApacheErrorParser < RegexpParser
|
22
|
+
Plugin.register_parser("apache_error", self)
|
23
|
+
config_set_default :expression, /^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/parser'
|
18
|
+
|
19
|
+
require 'csv'
|
20
|
+
|
21
|
+
module Fluent
|
22
|
+
module Plugin
|
23
|
+
class CSVParser < Parser
|
24
|
+
Plugin.register_parser('csv', self)
|
25
|
+
|
26
|
+
desc 'Names of fields included in each lines'
|
27
|
+
config_param :keys, :array, value_type: :string
|
28
|
+
desc 'The delimiter character (or string) of CSV values'
|
29
|
+
config_param :delimiter, :string, default: ','
|
30
|
+
desc 'The parser type used to parse CSV line'
|
31
|
+
config_param :parser_engine, :enum, list: [:normal, :fast], default: :normal, alias: :parser_type
|
32
|
+
|
33
|
+
def configure(conf)
|
34
|
+
super
|
35
|
+
|
36
|
+
|
37
|
+
if @parser_engine == :fast
|
38
|
+
@quote_char = '"'
|
39
|
+
@escape_pattern = Regexp.compile(@quote_char * 2)
|
40
|
+
|
41
|
+
m = method(:parse_fast)
|
42
|
+
self.singleton_class.module_eval do
|
43
|
+
define_method(:parse, m)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def parse(text, &block)
|
49
|
+
values = CSV.parse_line(text, col_sep: @delimiter)
|
50
|
+
r = Hash[@keys.zip(values)]
|
51
|
+
time, record = convert_values(parse_time(r), r)
|
52
|
+
yield time, record
|
53
|
+
end
|
54
|
+
|
55
|
+
def parse_fast(text, &block)
|
56
|
+
r = parse_fast_internal(text)
|
57
|
+
time, record = convert_values(parse_time(r), r)
|
58
|
+
yield time, record
|
59
|
+
end
|
60
|
+
|
61
|
+
# CSV.parse_line is too slow due to initialize lots of object and
|
62
|
+
# CSV module doesn't provide the efficient method for parsing single line.
|
63
|
+
# This method avoids the overhead of CSV.parse_line for typical patterns
|
64
|
+
def parse_fast_internal(text)
|
65
|
+
record = {}
|
66
|
+
text.chomp!
|
67
|
+
|
68
|
+
return record if text.empty?
|
69
|
+
|
70
|
+
# use while because while is now faster than each_with_index
|
71
|
+
columns = text.split(@delimiter, -1)
|
72
|
+
num_columns = columns.size
|
73
|
+
i = 0
|
74
|
+
j = 0
|
75
|
+
while j < num_columns
|
76
|
+
column = columns[j]
|
77
|
+
|
78
|
+
case column.count(@quote_char)
|
79
|
+
when 0
|
80
|
+
if column.empty?
|
81
|
+
column = nil
|
82
|
+
end
|
83
|
+
when 1
|
84
|
+
if column.start_with?(@quote_char)
|
85
|
+
to_merge = [column]
|
86
|
+
j += 1
|
87
|
+
while j < num_columns
|
88
|
+
merged_col = columns[j]
|
89
|
+
to_merge << merged_col
|
90
|
+
break if merged_col.end_with?(@quote_char)
|
91
|
+
j += 1
|
92
|
+
end
|
93
|
+
column = to_merge.join(@delimiter)[1..-2]
|
94
|
+
end
|
95
|
+
when 2
|
96
|
+
if column.start_with?(@quote_char) && column.end_with?(@quote_char)
|
97
|
+
column = column[1..-2]
|
98
|
+
end
|
99
|
+
else
|
100
|
+
if column.start_with?(@quote_char) && column.end_with?(@quote_char)
|
101
|
+
column = column[1..-2]
|
102
|
+
end
|
103
|
+
column.gsub!(@escape_pattern, @quote_char)
|
104
|
+
end
|
105
|
+
|
106
|
+
record[@keys[i]] = column
|
107
|
+
j += 1
|
108
|
+
i += 1
|
109
|
+
end
|
110
|
+
record
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/parser'
|
18
|
+
require 'fluent/time'
|
19
|
+
require 'fluent/oj_options'
|
20
|
+
|
21
|
+
require 'yajl'
|
22
|
+
require 'json'
|
23
|
+
|
24
|
+
module Fluent
|
25
|
+
module Plugin
|
26
|
+
class JSONParser < Parser
|
27
|
+
Plugin.register_parser('json', self)
|
28
|
+
|
29
|
+
config_set_default :time_key, 'time'
|
30
|
+
desc 'Set JSON parser'
|
31
|
+
config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
|
32
|
+
|
33
|
+
# The Yajl library defines a default buffer size of 8KiB when parsing
|
34
|
+
# from IO streams, so maintain this for backwards-compatibility.
|
35
|
+
# https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse
|
36
|
+
desc 'Set the buffer size that Yajl will use when parsing streaming input'
|
37
|
+
config_param :stream_buffer_size, :integer, default: 8192
|
38
|
+
|
39
|
+
config_set_default :time_type, :float
|
40
|
+
|
41
|
+
def configure(conf)
|
42
|
+
if conf.has_key?('time_format')
|
43
|
+
conf['time_type'] ||= 'string'
|
44
|
+
end
|
45
|
+
|
46
|
+
super
|
47
|
+
@load_proc, @error_class = configure_json_parser(@json_parser)
|
48
|
+
end
|
49
|
+
|
50
|
+
def configure_json_parser(name)
|
51
|
+
case name
|
52
|
+
when :oj
|
53
|
+
raise LoadError unless Fluent::OjOptions.available?
|
54
|
+
[Oj.method(:load), Oj::ParseError]
|
55
|
+
when :json then [JSON.method(:load), JSON::ParserError]
|
56
|
+
when :yajl then [Yajl.method(:load), Yajl::ParseError]
|
57
|
+
else
|
58
|
+
raise "BUG: unknown json parser specified: #{name}"
|
59
|
+
end
|
60
|
+
rescue LoadError => ex
|
61
|
+
name = :yajl
|
62
|
+
if log
|
63
|
+
if /\boj\z/ =~ ex.message
|
64
|
+
log.info "Oj is not installed, and failing back to Yajl for json parser"
|
65
|
+
else
|
66
|
+
log.warn ex.message
|
67
|
+
end
|
68
|
+
end
|
69
|
+
retry
|
70
|
+
end
|
71
|
+
|
72
|
+
def parse(text)
|
73
|
+
record = @load_proc.call(text)
|
74
|
+
time = parse_time(record)
|
75
|
+
if @execute_convert_values
|
76
|
+
time, record = convert_values(time, record)
|
77
|
+
end
|
78
|
+
yield time, record
|
79
|
+
rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
|
80
|
+
yield nil, nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def parser_type
|
84
|
+
:text
|
85
|
+
end
|
86
|
+
|
87
|
+
def parse_io(io, &block)
|
88
|
+
y = Yajl::Parser.new
|
89
|
+
y.on_parse_complete = ->(record){
|
90
|
+
block.call(parse_time(record), record)
|
91
|
+
}
|
92
|
+
y.parse(io, @stream_buffer_size)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|