fluent-plugin-elasticsearch 5.0.2 → 5.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa89d08181943944c6bda419dcf34ef9f1bbe94b67bb270c293d8a0fa03dced0
-  data.tar.gz: d276c5a35783d83608032740c69e2d33e9b6462f507b83e4ff393feb4ed1d151
+  metadata.gz: 6eb418d889b91bf79c37c1cd72789981eb4133bcfc34e21b26e79bb919462272
+  data.tar.gz: 168ddc77fb73216da63f1ce65ed673cf12ab13d6db549fc3e2da81956c938990
 SHA512:
-  metadata.gz: 28c78fb744db4ae40a31f27a4d25fed52d278cfc8cf39c5d5bcbcf129302c1a2157e609179ecfa6693465454c131834bb04d6f40b42a46c6cdf45e71f3036291
-  data.tar.gz: f19746088603311f3aa06787fcc219e810ef97762ae72563b557a00ad29518a06f4c641a951913be3865b86cae7cda2329856c4d2317a0a8a2360109aa8ffca5
+  metadata.gz: 5c4a2a8f63b25ea8785e0d4ef291043a93a86fea8f42638caaf46dd3efe29fd0a12919dd339d8a28c97aef65494ba054a3ff3312a122ab2cd89cb54cb069055c
+  data.tar.gz: 6b13851ce29b2a6f2083ebc57ae733a9ca00d79a9fd09ac4c7630d99a9c6d2da302c48f2e0cd901810e0a851911e14f8355fc336e21afd97d3f33fe205b0c78a

data/History.md

@@ -1,6 +1,9 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+- Fix use_legacy_template documentation (#880)
+- Add FAQ for dynamic index/template (#878)
+- Handle IPv6 address string on host and hosts parameters (#877)
 
 ### 5.0.2
 - GitHub Actions: Tweak Ruby versions on test (#875)

data/README.Troubleshooting.md

@@ -10,6 +10,7 @@
   + [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
   + [Fluentd seems to hang if it unable to connect Elasticsearch, why?](#fluentd-seems-to-hang-if-it-unable-to-connect-elasticsearch-why)
   + [Enable Index Lifecycle Management](#enable-index-lifecycle-management)
+    + [Configuring for dynamic index or template](#configuring-for-dynamic-index-or-template)
   + [How to specify index codec](#how-to-specify-index-codec)
   + [Cannot push logs to Elasticsearch with connect_write timeout reached, why?](#cannot-push-logs-to-elasticsearch-with-connect_write-timeout-reached-why)
 
@@ -524,6 +525,96 @@ template_name your-fluentd-template
 template_file /path/to/fluentd-template.json
 ```
 
+#### Configuring for dynamic index or template
+
+Some users want to setup ILM for dynamic index/template.
+`index_petterns` and `template.settings.index.lifecycle.name` in Elasticsearch template will be overwritten by Elasticsearch plugin:
+
+```json
+{
+  "index_patterns": ["mock"],
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "mock",
+          "rollover_alias": "mock"
+        },
+        "number_of_shards": "<<shard>>",
+        "number_of_replicas": "<<replica>>"
+      }
+    }
+  }
+}
+```
+
+This template will be handled with:
+
+```aconf
+<source>
+  @type http
+  port 5004
+  bind 0.0.0.0
+  body_size_limit 32m
+  keepalive_timeout 10s
+  <parse>
+    @type json
+  </parse>
+</source>
+
+<match kubernetes.var.log.containers.**etl-webserver**.log>
+    @type elasticsearch
+    @id out_es_etl_webserver
+    @log_level info
+    include_tag_key true
+    host $HOST
+    port $PORT
+    path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}"
+    request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '30s'}"
+    scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
+    ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
+    ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}"
+    reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}"
+    reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}"
+    reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}"
+    log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}"
+    logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'etl-webserver'}"
+    logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'false'}"
+    index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'etl-webserver'}"
+    type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}"
+    time_key "#{ENV['FLUENT_ELASTICSEARCH_TIME_KEY'] || '@timestamp'}"
+    include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'true'}"
+
+    # ILM Settings - WITH ROLLOVER support
+    # https://github.com/uken/fluent-plugin-elasticsearch#enable-index-lifecycle-management
+    application_name "etl-webserver"
+    index_date_pattern ""
+    # Policy configurations
+    enable_ilm true
+    ilm_policy_id etl-webserver
+    ilm_policy_overwrite true
+    ilm_policy {"policy": {"phases": {"hot": {"min_age": "0ms","actions": {"rollover": {"max_age": "5m","max_size": "3gb"},"set_priority": {"priority": 100}}},"delete": {"min_age": "30d","actions": {"delete": {"delete_searchable_snapshot": true}}}}}}
+    use_legacy_template false
+    template_name etl-webserver
+    template_file /configs/index-template.json
+    template_overwrite true
+    customize_template {"<<shard>>": "3","<<replica>>": "0"}
+
+    <buffer>
+        flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
+        flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
+        chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '8MB'}"
+        total_limit_size "#{ENV['FLUENT_ELASTICSEARCH_TOTAL_LIMIT_SIZE'] || '450MB'}"
+        queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
+        retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '60s'}"
+        retry_forever false
+    </buffer>
+</match>
+```
+
+For more details, please refer the discussion:
+https://github.com/uken/fluent-plugin-elasticsearch/issues/867
+
 ### How to specify index codec
 
 Elasticsearch can handle compression methods for stored data such as LZ4 and best_compression.

data/README.md

@@ -171,6 +171,24 @@ You can specify Elasticsearch host by this parameter.
 
 **Note:** Since v3.3.2, `host` parameter supports builtin placeholders. If you want to send events dynamically into different hosts at runtime with `elasticsearch_dynamic` output plugin, please consider to switch to use plain `elasticsearch` output plugin. In more detail for builtin placeholders, please refer to [Placeholders](#placeholders) section.
 
+To use IPv6 address on `host` parameter, you can use the following styles:
+
+#### string style
+
+To use string style, you must quote IPv6 address due to prevent to be interpreted as JSON:
+
+```
+host "[2404:7a80:d440:3000:192a:a292:bd7f:ca10]"
+```
+
+#### raw style
+
+You can also specify raw IPv6 address. This will be handled as `[specified IPv6 address]`:
+
+```
+host 2404:7a80:d440:3000:192a:a292:bd7f:ca10
+```
+
 ### port
 
 ```
@@ -237,6 +255,16 @@ hosts host1:port1,host2:port2,host3 # port3 is 9200
 
 **Note:** Up until v2.8.5, it was allowed to embed the username/password in the URL. However, this syntax is deprecated as of v2.8.6 because it was found to cause serious connection problems (See #394). Please migrate your settings to use the `user` and `password` field (described below) instead.
 
+#### IPv6 addresses
+
+When you want to specify IPv6 addresses, you must specify schema together:
+
+```
+hosts http://[2404:7a80:d440:3000:de:7311:6329:2e6c]:port1,http://[2404:7a80:d440:3000:de:7311:6329:1e6c]:port2,http://[2404:7a80:d440:3000:de:6311:6329:2e6c]:port3
+```
+
+If you don't specify hosts with schema together, Elasticsearch plugin complains Invalid URI for them.
+
 ### user, password, path, scheme, ssl_verify
 
 ```
@@ -1325,9 +1353,9 @@ Default value is `nil`.
 
 Use legacy template or not.
 
-Elasticsearch 7.8 or later supports the brand new composable templates.
+For Elasticsearch 7.8 or later, users can specify this parameter as `false` if their [template_file](#template_file) contains a composable index template.
 
-For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
+For Elasticsearch 7.7 or older, users should specify this parameter as `true`.
 
 Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.0.2'
+  s.version       = '5.0.3'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -13,6 +13,7 @@ begin
   require 'strptime'
 rescue LoadError
 end
+require 'resolv'
 
 require 'fluent/plugin/output'
 require 'fluent/event'
@@ -668,7 +669,11 @@ EOC
           end
         end.compact
       else
-        [{host: @host, port: @port, scheme: @scheme.to_s}]
+        if Resolv::IPv6::Regex.match(@host)
+          [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
+        else
+          [{host: @host, port: @port, scheme: @scheme.to_s}]
+        end
       end.each do |host|
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path

data/test/plugin/test_out_elasticsearch.rb

@@ -3612,6 +3612,74 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  class IPv6AdressStringHostsTest < self
+    def test_legacy_hosts_list
+      config = %{
+        hosts    "[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:50,host2:100,host3"
+        scheme   https
+        path     /es/
+        port     123
+      }
+      instance = driver(config).instance
+
+      assert_raise(URI::InvalidURIError) do
+        instance.get_connection_options[:hosts].length
+      end
+    end
+
+    def test_hosts_list
+      config = %{
+        hosts    https://john:password@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
+        path     /default_path
+        user     default_user
+        password default_password
+      }
+      instance = driver(config).instance
+
+      assert_equal 2, instance.get_connection_options[:hosts].length
+      host1, host2 = instance.get_connection_options[:hosts]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 443, host1[:port]
+      assert_equal 'https', host1[:scheme]
+      assert_equal 'john', host1[:user]
+      assert_equal 'password', host1[:password]
+      assert_equal '/elastic/', host1[:path]
+
+      assert_equal 'host2', host2[:host]
+      assert_equal 'http', host2[:scheme]
+      assert_equal 'default_user', host2[:user]
+      assert_equal 'default_password', host2[:password]
+      assert_equal '/default_path', host2[:path]
+    end
+
+    def test_hosts_list_with_escape_placeholders
+      config = %{
+        hosts    https://%{j+hn}:%{passw@rd}@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
+        path     /default_path
+        user     default_user
+        password default_password
+      }
+      instance = driver(config).instance
+
+      assert_equal 2, instance.get_connection_options[:hosts].length
+      host1, host2 = instance.get_connection_options[:hosts]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 443, host1[:port]
+      assert_equal 'https', host1[:scheme]
+      assert_equal 'j%2Bhn', host1[:user]
+      assert_equal 'passw%40rd', host1[:password]
+      assert_equal '/elastic/', host1[:path]
+
+      assert_equal 'host2', host2[:host]
+      assert_equal 'http', host2[:scheme]
+      assert_equal 'default_user', host2[:user]
+      assert_equal 'default_password', host2[:password]
+      assert_equal '/default_path', host2[:path]
+    end
+  end
+
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
@@ -3665,6 +3733,46 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert(ports.none? { |p| p == 9200 })
   end
 
+  class IPv6AdressStringHostTest < self
+    def test_single_host_params_and_defaults
+      config = %{
+        host     2404:7a80:d440:3000:192a:a292:bd7f:ca19
+        user     john
+        password doe
+      }
+      instance = driver(config).instance
+
+      assert_equal 1, instance.get_connection_options[:hosts].length
+      host1 = instance.get_connection_options[:hosts][0]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 9200, host1[:port]
+      assert_equal 'http', host1[:scheme]
+      assert_equal 'john', host1[:user]
+      assert_equal 'doe', host1[:password]
+      assert_equal nil, host1[:path]
+    end
+
+    def test_single_host_params_and_defaults_with_escape_placeholders
+      config = %{
+        host     2404:7a80:d440:3000:192a:a292:bd7f:ca19
+        user     %{j+hn}
+        password %{d@e}
+      }
+      instance = driver(config).instance
+
+      assert_equal 1, instance.get_connection_options[:hosts].length
+      host1 = instance.get_connection_options[:hosts][0]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 9200, host1[:port]
+      assert_equal 'http', host1[:scheme]
+      assert_equal 'j%2Bhn', host1[:user]
+      assert_equal 'd%40e', host1[:password]
+      assert_equal nil, host1[:path]
+    end
+  end
+
   def test_password_is_required_if_specify_user
     config = %{
       user john

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.0.2
+  version: 5.0.3
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-19 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd