fluent-plugin-elasticsearch 5.0.0 → 5.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f2f8268d9a8a5acf6d941a915044bd3781a5722cea8bc7ac205d6b7fd6fe580
-  data.tar.gz: 7d9373fb963040efac0ea42f11bf1841f2e0b2a39b5566f5e2794507e77e5f5c
+  metadata.gz: 0e7a5cd238a268dd3f141ce4ad1dedfacbacc98fdd22f73839bb0e1cdbccc586
+  data.tar.gz: 8077a55fae311fd0bd9cd84fc2b2eb34fad8f7a2a890b2707d37c7ac1c753975
 SHA512:
-  metadata.gz: 0e432748181717cedfa55239d2da7b0141c3280da7240f6f9643db411f1f3168f00f88c6933d65a5d1b944620dab0d87ce088f4dba76fdf17cec336ca55e83bf
-  data.tar.gz: f44b2a14c5a13e1a59bd7d6926af13a9a9ba7b990594bca45de20071756d9ca88f7ee53d6ec00e09f0ad2ed054510a7803ac11c7b695275d37dbacb1e5d51457
+  metadata.gz: 7bede6c6213db75e2636c6bf31e65b3256ae709ef600b9f3fca8973943344e07421daf4ed04fc0f3dd06f6b1f662c93c27d9240a323f8139aee3bf2a99ed0f9b
+  data.tar.gz: caed0a434434c903451cdd4b92ae4427ca39984a8405154308d70a2b68351c6a05b7e505c08cd776230cef9dbbda46f4256cd7de138f3549a1860b5b6cee8e3c

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,8 +2,29 @@
 
 ### [Unreleased]
 
+### 5.0.5
+- Drop json_parse_exception messages for bulk failures (#900)
+- GitHub Actions: Drop Ruby 2.5 due to EOL (#894)
+
+### 5.0.4
+- test: out_elasticsearch: Remove a needless headers from affinity stub (#888)
+- Target Index Affinity (#883)
+
+### 5.0.3
+- Fix use_legacy_template documentation (#880)
+- Add FAQ for dynamic index/template (#878)
+- Handle IPv6 address string on host and hosts parameters (#877)
+
+### 5.0.2
+- GitHub Actions: Tweak Ruby versions on test (#875)
+- test: datastreams: Set nonexistent datastream as default (#874)
+- Fix overwriting of index template and index lifecycle policy on existing data streams (#872)
+
+### 5.0.1
+- Use elasticsearch/api instead of elasticsearch/xpack (#870)
+
 ### 5.0.0
--  Support #retry_operate on data stream (#863)
+- Support #retry_operate on data stream (#863)
 - Support placeholder in @data\_stream\_name for @type elasticsearch\_data\_stream (#862)
 - Extract troubleshooting section (#861)
 - Fix unmatched `<source>` close tag (#860)

data/README.Troubleshooting.md

@@ -10,6 +10,7 @@
   + [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
   + [Fluentd seems to hang if it unable to connect Elasticsearch, why?](#fluentd-seems-to-hang-if-it-unable-to-connect-elasticsearch-why)
   + [Enable Index Lifecycle Management](#enable-index-lifecycle-management)
+    + [Configuring for dynamic index or template](#configuring-for-dynamic-index-or-template)
   + [How to specify index codec](#how-to-specify-index-codec)
   + [Cannot push logs to Elasticsearch with connect_write timeout reached, why?](#cannot-push-logs-to-elasticsearch-with-connect_write-timeout-reached-why)
 
@@ -524,6 +525,96 @@ template_name your-fluentd-template
 template_file /path/to/fluentd-template.json
 ```
 
+#### Configuring for dynamic index or template
+
+Some users want to setup ILM for dynamic index/template.
+`index_petterns` and `template.settings.index.lifecycle.name` in Elasticsearch template will be overwritten by Elasticsearch plugin:
+
+```json
+{
+  "index_patterns": ["mock"],
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "mock",
+          "rollover_alias": "mock"
+        },
+        "number_of_shards": "<<shard>>",
+        "number_of_replicas": "<<replica>>"
+      }
+    }
+  }
+}
+```
+
+This template will be handled with:
+
+```aconf
+<source>
+  @type http
+  port 5004
+  bind 0.0.0.0
+  body_size_limit 32m
+  keepalive_timeout 10s
+  <parse>
+    @type json
+  </parse>
+</source>
+
+<match kubernetes.var.log.containers.**etl-webserver**.log>
+    @type elasticsearch
+    @id out_es_etl_webserver
+    @log_level info
+    include_tag_key true
+    host $HOST
+    port $PORT
+    path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}"
+    request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '30s'}"
+    scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
+    ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
+    ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}"
+    reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}"
+    reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}"
+    reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}"
+    log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}"
+    logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'etl-webserver'}"
+    logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'false'}"
+    index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'etl-webserver'}"
+    type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}"
+    time_key "#{ENV['FLUENT_ELASTICSEARCH_TIME_KEY'] || '@timestamp'}"
+    include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'true'}"
+
+    # ILM Settings - WITH ROLLOVER support
+    # https://github.com/uken/fluent-plugin-elasticsearch#enable-index-lifecycle-management
+    application_name "etl-webserver"
+    index_date_pattern ""
+    # Policy configurations
+    enable_ilm true
+    ilm_policy_id etl-webserver
+    ilm_policy_overwrite true
+    ilm_policy {"policy": {"phases": {"hot": {"min_age": "0ms","actions": {"rollover": {"max_age": "5m","max_size": "3gb"},"set_priority": {"priority": 100}}},"delete": {"min_age": "30d","actions": {"delete": {"delete_searchable_snapshot": true}}}}}}
+    use_legacy_template false
+    template_name etl-webserver
+    template_file /configs/index-template.json
+    template_overwrite true
+    customize_template {"<<shard>>": "3","<<replica>>": "0"}
+
+    <buffer>
+        flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
+        flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
+        chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '8MB'}"
+        total_limit_size "#{ENV['FLUENT_ELASTICSEARCH_TOTAL_LIMIT_SIZE'] || '450MB'}"
+        queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
+        retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '60s'}"
+        retry_forever false
+    </buffer>
+</match>
+```
+
+For more details, please refer the discussion:
+https://github.com/uken/fluent-plugin-elasticsearch/issues/867
+
 ### How to specify index codec
 
 Elasticsearch can handle compression methods for stored data such as LZ4 and best_compression.

data/README.md

@@ -38,6 +38,7 @@ Current maintainers: @cosmo0920
   + [suppress_type_name](#suppress_type_name)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
+  + [target_index_affinity](#target_index_affinity)
   + [template_name](#template_name)
   + [template_file](#template_file)
   + [template_overwrite](#template_overwrite)
@@ -171,6 +172,24 @@ You can specify Elasticsearch host by this parameter.
 
 **Note:** Since v3.3.2, `host` parameter supports builtin placeholders. If you want to send events dynamically into different hosts at runtime with `elasticsearch_dynamic` output plugin, please consider to switch to use plain `elasticsearch` output plugin. In more detail for builtin placeholders, please refer to [Placeholders](#placeholders) section.
 
+To use IPv6 address on `host` parameter, you can use the following styles:
+
+#### string style
+
+To use string style, you must quote IPv6 address due to prevent to be interpreted as JSON:
+
+```
+host "[2404:7a80:d440:3000:192a:a292:bd7f:ca10]"
+```
+
+#### raw style
+
+You can also specify raw IPv6 address. This will be handled as `[specified IPv6 address]`:
+
+```
+host 2404:7a80:d440:3000:192a:a292:bd7f:ca10
+```
+
 ### port
 
 ```
@@ -237,6 +256,16 @@ hosts host1:port1,host2:port2,host3 # port3 is 9200
 
 **Note:** Up until v2.8.5, it was allowed to embed the username/password in the URL. However, this syntax is deprecated as of v2.8.6 because it was found to cause serious connection problems (See #394). Please migrate your settings to use the `user` and `password` field (described below) instead.
 
+#### IPv6 addresses
+
+When you want to specify IPv6 addresses, you must specify schema together:
+
+```
+hosts http://[2404:7a80:d440:3000:de:7311:6329:2e6c]:port1,http://[2404:7a80:d440:3000:de:7311:6329:1e6c]:port2,http://[2404:7a80:d440:3000:de:6311:6329:2e6c]:port3
+```
+
+If you don't specify hosts with schema together, Elasticsearch plugin complains Invalid URI for them.
+
 ### user, password, path, scheme, ssl_verify
 
 ```
@@ -426,6 +455,75 @@ and this record will be written to the specified index (`logstash-2014.12.19`) r
 
 Similar to `target_index_key` config, find the type name to write to in the record under this key (or nested record). If key not found in record - fallback to `type_name` (default "fluentd").
 
+### target_index_affinity
+
+Enable plugin to dynamically select logstash time based target index in update/upsert operations based on already indexed data rather than current time of indexing.
+
+```
+target_index_affinity true # defaults to false
+```
+
+By default plugin writes data of logstash format index based on current time. For example daily based index after mignight data is written to newly created index. This is normally ok when data is coming from single source and not updated after indexing.
+
+But if you have a use case where data is also updated after indexing and `id_key` is used to identify the document uniquely for updating. Logstash format is wanted to be used for easy data managing and retention. Updates are done right after indexing to complete the data (all data not available from single source) and no updates are done anymore later point on time. In this case problem happends at index rotation time where write to 2 indexes with same id_key value may happen.
+
+This setting will search existing data by using elastic search's [id query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-ids-query.html) using `id_key` value (with logstash_prefix and logstash_prefix_separator index pattarn e.g. `logstash-*`). The index of found data is used for update/upsert. When no data is found, data is written to current logstash index as normally.
+
+This setting requires following other settings:
+```
+logstash_format true
+id_key myId  # Some field on your data to identify the data uniquely
+write_operation upsert  # upsert or update
+```
+
+Suppose you have the following situation where you have 2 different match to consume data from 2 different Kafka topics independently but close in time with each other (order not known).
+
+```
+  <match data1>
+    @type elasticsearch
+    ...
+    id_key myId
+    write_operation upsert
+    logstash_format true
+    logstash_dateformat %Y.%m.%d
+    logstash_prefix myindexprefix
+    target_index_affinity true
+    ...
+
+  <match data2>
+    @type elasticsearch
+    ...
+    id_key myId
+    write_operation upsert
+    logstash_format true
+    logstash_dateformat %Y.%m.%d
+    logstash_prefix myindexprefix
+    target_index_affinity true
+    ...
+```
+
+If your first (data1) input is:
+```
+{
+  "myId": "myuniqueId1",
+  "datafield1": "some value",
+}
+```
+
+and your second (data2) input is:
+```
+{
+  "myId": "myuniqueId1",
+  "datafield99": "some important data from other source tightly related to id myuniqueId1 and wanted to be in same document.",
+}
+```
+
+Date today is 10.05.2021 so data is written to index `myindexprefix-2021.05.10` when both data1 and data2 is consumed during today.
+But when we are close to index rotation and data1 is consumed and indexed at `2021-05-10T23:59:55.59707672Z` and data2
+is consumed a bit later at `2021-05-11T00:00:58.222079Z` i.e. logstash index has been rotated and normally data2 would have been written
+to index `myindexprefix-2021.05.11`. But with target_index_affinity setting as value true, data2 is now written to index `myindexprefix-2021.05.10`
+into same document with data1 as wanted and duplicated document is avoided.
+
 ### template_name
 
 The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
@@ -1325,9 +1423,9 @@ Default value is `nil`.
 
 Use legacy template or not.
 
-Elasticsearch 7.8 or later supports the brand new composable templates.
+For Elasticsearch 7.8 or later, users can specify this parameter as `false` if their [template_file](#template_file) contains a composable index template.
 
-For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
+For Elasticsearch 7.7 or older, users should specify this parameter as `true`.
 
 Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.0.0'
+  s.version       = '5.0.5'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
 
 
   s.add_development_dependency 'rake', '>= 0'
+  s.add_development_dependency 'webrick', '~> 1.7.0'
   s.add_development_dependency 'webmock', '~> 3'
   s.add_development_dependency 'test-unit', '~> 3.3.0'
   s.add_development_dependency 'minitest', '~> 5.8'

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -23,6 +23,10 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     unrecoverable_error_types.include?(type)
   end
 
+  def unrecoverable_record_error?(type)
+    ['json_parse_exception'].include?(type)
+  end
+
   def log_es_400_reason(&block)
     if @plugin.log_es_400_reason
       block.call
@@ -43,15 +47,17 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     stats = Hash.new(0)
     meta = {}
     header = {}
+    affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
     chunk.msgpack_each do |time, rawrecord|
       bulk_message = ''
       next unless rawrecord.is_a? Hash
       begin
         # we need a deep copy for process_message to alter
         processrecord = Marshal.load(Marshal.dump(rawrecord))
-        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
         next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
       rescue => e
+        @plugin.log.debug("Exception in error handler during deep copy: #{e}")
         stats[:bad_chunk_record] += 1
         next
       end
@@ -105,10 +111,15 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
           stats[type] += 1
-          retry_stream.add(time, rawrecord)
           if unrecoverable_error?(type)
             raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
           end
+          if unrecoverable_record_error?(type)
+            @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - #{type}: #{reason}"))
+            next
+          else
+            retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
+          end
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -2,6 +2,7 @@
 require 'date'
 require 'excon'
 require 'elasticsearch'
+require 'set'
 begin
   require 'elasticsearch/xpack'
 rescue LoadError
@@ -13,6 +14,7 @@ begin
   require 'strptime'
 rescue LoadError
 end
+require 'resolv'
 
 require 'fluent/plugin/output'
 require 'fluent/event'
@@ -174,6 +176,7 @@ EOC
     config_param :truncate_caches_interval, :time, :default => nil
     config_param :use_legacy_template, :bool, :default => true
     config_param :catch_transport_exception_on_retry, :bool, :default => true
+    config_param :target_index_affinity, :bool, :default => false
 
     config_section :metadata, param_name: :metainfo, multi: false do
       config_param :include_chunk_id, :bool, :default => false
@@ -668,7 +671,11 @@ EOC
           end
         end.compact
       else
-        [{host: @host, port: @port, scheme: @scheme.to_s}]
+        if Resolv::IPv6::Regex.match(@host)
+          [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
+        else
+          [{host: @host, port: @port, scheme: @scheme.to_s}]
+        end
       end.each do |host|
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path
@@ -829,13 +836,14 @@ EOC
                extract_placeholders(@host, chunk)
              end
 
+      affinity_target_indices = get_affinity_target_indices(chunk)
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
         record = inject_chunk_id_to_record_if_needed(record, chunk_id)
 
         begin
-          meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
+          meta, header, record = process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
           info = if @include_index_in_url
                    RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
                  else
@@ -872,6 +880,42 @@ EOC
       end
     end
 
+    def target_index_affinity_enabled?()
+      @target_index_affinity && @logstash_format && @id_key && (@write_operation == UPDATE_OP || @write_operation == UPSERT_OP)
+    end
+
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      if target_index_affinity_enabled?()
+        id_key_accessor = record_accessor_create(@id_key)
+        ids = Set.new
+        chunk.msgpack_each do |time, record|
+          next unless record.is_a? Hash
+          begin
+            ids << id_key_accessor.call(record)
+          end
+        end
+        log.debug("Find affinity target_indices by quering on ES (write_operation #{@write_operation}) for ids: #{ids.to_a}")
+        options = {
+          :index => "#{logstash_prefix}#{@logstash_prefix_separator}*",
+        }
+        query = {
+          'query' => { 'ids' => { 'values' => ids.to_a } },
+          '_source' => false,
+          'sort' => [
+            {"_index" => {"order" => "desc"}}
+         ]
+        }
+        result = client.search(options.merge(:body => Yajl.dump(query)))
+        # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
+        result['hits']['hits'].each do |hit|
+          indices[hit["_id"]] = hit["_index"]
+          log.debug("target_index for id: #{hit["_id"]} from es: #{hit["_index"]}")
+        end
+      end
+      indices
+    end
+
     def split_request?(bulk_message, info)
       # For safety.
     end
@@ -884,7 +928,7 @@ EOC
       false
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
 
       if @flatten_hashes
@@ -925,6 +969,15 @@ EOC
         record[@tag_key] = tag
       end
 
+      # If affinity target indices map has value for this particular id, use it as target_index
+      if !affinity_target_indices.empty?
+        id_accessor = record_accessor_create(@id_key)
+        id_value = id_accessor.call(record)
+        if affinity_target_indices.key?(id_value)
+          target_index = affinity_target_indices[id_value]
+        end
+      end
+
       target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
       if target_type_parent && target_type_parent[target_type_child_key]
         target_type = target_type_parent.delete(target_type_child_key)

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -18,9 +18,10 @@ module Fluent::Plugin
       super
 
       begin
+        require 'elasticsearch/api'
         require 'elasticsearch/xpack'
       rescue LoadError
-        raise Fluent::ConfigError, "'elasticsearch/xpack'' is required for <@elasticsearch_data_stream>."
+        raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
       end
 
       # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
@@ -66,6 +67,7 @@ module Fluent::Plugin
     end
 
     def create_ilm_policy(name)
+      return if data_stream_exist?(name)
       params = {
         policy_id: "#{name}_policy",
         body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
@@ -78,6 +80,7 @@ module Fluent::Plugin
     end
 
     def create_index_template(name)
+      return if data_stream_exist?(name)
       body = {
         "index_patterns" => ["#{name}*"],
         "data_stream" => {},

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -27,13 +27,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
        @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       return [meta, header, record]
     end
 
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      indices
+    end
+
     def append_record_to_messages(op, meta, header, record, msgs)
       if record.has_key?('raise') && record['raise']
-        raise Exception('process_message')
+        raise 'process_message'
       end
       return true
     end
@@ -302,7 +307,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
   def test_retry_error
     records = []
     error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
+    error_records.merge!({0=>true, 4=>true})
     10.times do |i|
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
@@ -386,6 +391,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
               "reason":"unrecognized error"
             }
           }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "9",
+            "status" : 500,
+            "error" : {
+              "type" : "json_parse_exception",
+              "reason":"Invalid UTF-8 start byte 0x92\\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@204fe9c9; line: 1, column: 81]"
+            }
+          }
         }
       ]
     }))
@@ -400,12 +417,12 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
         next unless e.respond_to?(:retry_stream)
         e.retry_stream.each {|time, record| records << record}
       end
-      assert_equal 2, records.length
-      assert_equal 2, records[0]['_id']
-      assert_equal 8, records[1]['_id']
+      assert_equal 2, records.length, "Exp. retry_stream to contain records"
+      assert_equal 2, records[0]['_id'], "Exp record with given ID to in retry_stream"
+      assert_equal 8, records[1]['_id'], "Exp record with given ID to in retry_stream"
       error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 3, error_ids.length
-      assert_equal [5, 6, 7], error_ids
+      assert_equal 4, error_ids.length, "Exp. a certain number of records to be dropped from retry_stream"
+      assert_equal [5, 6, 7, 9], error_ids, "Exp. specific records to be dropped from retry_stream"
       @plugin.error_events.collect {|h| h[:error]}.each do |e|
         assert_true e.respond_to?(:backtrace)
       end

data/test/plugin/test_out_elasticsearch.rb

@@ -10,7 +10,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   include FlexMock::TestCase
   include Fluent::Test::Helpers
 
-  attr_accessor :index_cmds, :index_command_counts
+  attr_accessor :index_cmds, :index_command_counts, :index_cmds_all_requests
 
   def setup
     Fluent::Test.setup
@@ -70,6 +70,14 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
+  def stub_elastic_all_requests(url="http://localhost:9200/_bulk")
+    @index_cmds_all_requests = Array.new
+    stub_request(:post, url).with do |req|
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_cmds_all_requests << @index_cmds
+    end
+  end
+
   def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
     stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
   end
@@ -3612,6 +3620,74 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  class IPv6AdressStringHostsTest < self
+    def test_legacy_hosts_list
+      config = %{
+        hosts    "[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:50,host2:100,host3"
+        scheme   https
+        path     /es/
+        port     123
+      }
+      instance = driver(config).instance
+
+      assert_raise(URI::InvalidURIError) do
+        instance.get_connection_options[:hosts].length
+      end
+    end
+
+    def test_hosts_list
+      config = %{
+        hosts    https://john:password@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
+        path     /default_path
+        user     default_user
+        password default_password
+      }
+      instance = driver(config).instance
+
+      assert_equal 2, instance.get_connection_options[:hosts].length
+      host1, host2 = instance.get_connection_options[:hosts]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 443, host1[:port]
+      assert_equal 'https', host1[:scheme]
+      assert_equal 'john', host1[:user]
+      assert_equal 'password', host1[:password]
+      assert_equal '/elastic/', host1[:path]
+
+      assert_equal 'host2', host2[:host]
+      assert_equal 'http', host2[:scheme]
+      assert_equal 'default_user', host2[:user]
+      assert_equal 'default_password', host2[:password]
+      assert_equal '/default_path', host2[:path]
+    end
+
+    def test_hosts_list_with_escape_placeholders
+      config = %{
+        hosts    https://%{j+hn}:%{passw@rd}@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
+        path     /default_path
+        user     default_user
+        password default_password
+      }
+      instance = driver(config).instance
+
+      assert_equal 2, instance.get_connection_options[:hosts].length
+      host1, host2 = instance.get_connection_options[:hosts]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 443, host1[:port]
+      assert_equal 'https', host1[:scheme]
+      assert_equal 'j%2Bhn', host1[:user]
+      assert_equal 'passw%40rd', host1[:password]
+      assert_equal '/elastic/', host1[:path]
+
+      assert_equal 'host2', host2[:host]
+      assert_equal 'http', host2[:scheme]
+      assert_equal 'default_user', host2[:user]
+      assert_equal 'default_password', host2[:password]
+      assert_equal '/default_path', host2[:path]
+    end
+  end
+
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
@@ -3665,6 +3741,46 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert(ports.none? { |p| p == 9200 })
   end
 
+  class IPv6AdressStringHostTest < self
+    def test_single_host_params_and_defaults
+      config = %{
+        host     2404:7a80:d440:3000:192a:a292:bd7f:ca19
+        user     john
+        password doe
+      }
+      instance = driver(config).instance
+
+      assert_equal 1, instance.get_connection_options[:hosts].length
+      host1 = instance.get_connection_options[:hosts][0]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 9200, host1[:port]
+      assert_equal 'http', host1[:scheme]
+      assert_equal 'john', host1[:user]
+      assert_equal 'doe', host1[:password]
+      assert_equal nil, host1[:path]
+    end
+
+    def test_single_host_params_and_defaults_with_escape_placeholders
+      config = %{
+        host     2404:7a80:d440:3000:192a:a292:bd7f:ca19
+        user     %{j+hn}
+        password %{d@e}
+      }
+      instance = driver(config).instance
+
+      assert_equal 1, instance.get_connection_options[:hosts].length
+      host1 = instance.get_connection_options[:hosts][0]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 9200, host1[:port]
+      assert_equal 'http', host1[:scheme]
+      assert_equal 'j%2Bhn', host1[:user]
+      assert_equal 'd%40e', host1[:password]
+      assert_equal nil, host1[:path]
+    end
+  end
+
   def test_password_is_required_if_specify_user
     config = %{
       user john
@@ -3986,6 +4102,185 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal(pipeline, index_cmds.first['index']['pipeline'])
   end
 
+  def stub_elastic_affinity_target_index_search_with_body(url="http://localhost:9200/logstash-*/_search", ids, return_body_str)
+    # Note: ids used in query is unique list of ids
+    stub_request(:post, url)
+      .with(
+        body: "{\"query\":{\"ids\":{\"values\":#{ids.uniq.to_json}}},\"_source\":false,\"sort\":[{\"_index\":{\"order\":\"desc\"}}]}",
+      )
+      .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => return_body_str
+      }
+    end)
+  end
+
+  def stub_elastic_affinity_target_index_search(url="http://localhost:9200/logstash-*/_search", ids, indices)
+    # Example ids and indices arrays.
+    #  [ "3408a2c8eecd4fbfb82e45012b54fa82", "2816fc6ef4524b3f8f7e869002005433"]
+    #  [ "logstash-2021.04.28", "logstash-2021.04.29"]
+    body = %({
+      "took" : 31,
+      "timed_out" : false,
+      "_shards" : {
+        "total" : 52,
+        "successful" : 52,
+        "skipped" : 48,
+        "failed" : 0
+      },
+      "hits" : {
+        "total" : {
+          "value" : 356,
+          "relation" : "eq"
+        },
+        "max_score" : null,
+        "hits" : [
+          {
+            "_index" : "#{indices[0]}",
+            "_type" : "_doc",
+            "_id" : "#{ids[0]}",
+            "_score" : null,
+            "sort" : [
+              "#{indices[0]}"
+            ]
+          },
+          {
+            "_index" : "#{indices[1]}",
+            "_type" : "_doc",
+            "_id" : "#{ids[1]}",
+            "_score" : null,
+            "sort" : [
+              "#{indices[1]}"
+            ]
+          }
+        ]
+      }
+    })
+    stub_elastic_affinity_target_index_search_with_body(ids, body)
+  end
+
+  def stub_elastic_affinity_target_index_search_return_empty(url="http://localhost:9200/logstash-*/_search", ids)
+    empty_body = %({
+      "took" : 5,
+      "timed_out" : false,
+      "_shards" : {
+        "total" : 54,
+        "successful" : 54,
+        "skipped" : 53,
+        "failed" : 0
+      },
+      "hits" : {
+        "total" : {
+          "value" : 0,
+          "relation" : "eq"
+        },
+        "max_score" : null,
+        "hits" : [ ]
+      }
+    })
+    stub_elastic_affinity_target_index_search_with_body(ids, empty_body)
+  end
+
+  def test_writes_to_affinity_target_index
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    indices = ["logstash-2021.04.28"]
+    stub_elastic
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_write_operation_upsert
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation upsert")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    indices = ["logstash-2021.04.28"]
+    stub_elastic
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_index_not_exists_yet
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    stub_elastic
+    stub_elastic_affinity_target_index_search_return_empty(ids)
+    time = Time.parse Date.today.iso8601
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record('my_id' => my_id_value))
+    end
+    assert_equal("logstash-#{time.utc.strftime("%Y.%m.%d")}", index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_multiple_indices
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "2816fc6ef4524b3f8f7e869002005433"
+    my_id_value2 = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value, my_id_value2]
+    indices = ["logstash-2021.04.29", "logstash-2021.04.28"]
+    stub_elastic_all_requests
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+      driver.feed(sample_record('my_id' => my_id_value2))
+    end
+    assert_equal(2, index_cmds_all_requests.count)
+    assert_equal('logstash-2021.04.29', index_cmds_all_requests[0].first['update']['_index'])
+    assert_equal(my_id_value, index_cmds_all_requests[0].first['update']['_id'])
+    assert_equal('logstash-2021.04.28', index_cmds_all_requests[1].first['update']['_index'])
+    assert_equal(my_id_value2, index_cmds_all_requests[1].first['update']['_id'])
+  end
+
+  def test_writes_to_affinity_target_index_same_id_dublicated_write_to_oldest_index
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "2816fc6ef4524b3f8f7e869002005433"
+    # It may happen than same id has inserted to two index while data inserted during rollover period
+    ids = [my_id_value, my_id_value]
+    # Simulate the used sorting here, as search sorts indices in DESC order to pick only oldest index per single _id
+    indices = ["logstash-2021.04.29", "logstash-2021.04.28"]
+
+    stub_elastic_all_requests
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+
+    assert_equal(1, index_cmds_all_requests.count)
+    assert_equal('logstash-2021.04.28', index_cmds_all_requests[0].first['update']['_index'])
+    assert_equal(my_id_value, index_cmds_all_requests[0].first['update']['_id'])
+  end
+
   class PipelinePlaceholdersTest < self
     def test_writes_to_default_index_with_pipeline_tag_placeholder
       pipeline = "fluentd-${tag}"

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -79,7 +79,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   end
 
   def stub_nonexistent_data_stream?(name="foo")
-    stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, Elasticsearch::Transport::Transport::Errors::NotFound])
+    stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
   end
 
   def stub_bulk_feed(name="foo")
@@ -94,7 +94,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   def stub_default(name="foo")
     stub_ilm_policy(name)
     stub_index_template(name)
-    stub_existent_data_stream?(name)
+    stub_nonexistent_data_stream?(name)
     stub_data_stream(name)
   end
 
@@ -207,12 +207,12 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
 
-  def test_nonexistent_data_stream
+  def test_existent_data_stream
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
     stub_ilm_policy
     stub_index_template
-    stub_nonexistent_data_stream?
+    stub_existent_data_stream?
     stub_data_stream
     conf = config_element(
       'ROOT', '', {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 5.0.5
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-02 00:00:00.000000000 Z
+date: 2021-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -68,6 +68,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -209,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector