fluent-plugin-elasticsearch 5.0.2 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa89d08181943944c6bda419dcf34ef9f1bbe94b67bb270c293d8a0fa03dced0
-  data.tar.gz: d276c5a35783d83608032740c69e2d33e9b6462f507b83e4ff393feb4ed1d151
+  metadata.gz: 89dfb07388fcdb941bd3dc13a9e481fcb2ecf8f4a0b3c828b6a539f13c90e8ee
+  data.tar.gz: ee8ac22c05d144076e62311e013fea483d72493d6066f87712b5c325f3c215e7
 SHA512:
-  metadata.gz: 28c78fb744db4ae40a31f27a4d25fed52d278cfc8cf39c5d5bcbcf129302c1a2157e609179ecfa6693465454c131834bb04d6f40b42a46c6cdf45e71f3036291
-  data.tar.gz: f19746088603311f3aa06787fcc219e810ef97762ae72563b557a00ad29518a06f4c641a951913be3865b86cae7cda2329856c4d2317a0a8a2360109aa8ffca5
+  metadata.gz: a69fbfa9c0fab591f55656d87cc99aa003aa4e3b7a7d1b2819bd385e70d0c67d7b91dfff0fa42cfeafe94b6fe8f467b069ed075a2776809d0615f6e243f8c355
+  data.tar.gz: 572585eeb9e939d2b08e5fe0590bd25f02b8390a730d9f3012c44604b33123d9372cf59ac50a108119b413f3d718e2a606a0b21dc62f676199329e3b12c6073c

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,6 +2,23 @@
 
 ### [Unreleased]
 
+### 5.1.0
+- Correct default target bytes value (#914)
+- Handle elasticsearch-ruby 7.14 properly (#913)
+
+### 5.0.5
+- Drop json_parse_exception messages for bulk failures (#900)
+- GitHub Actions: Drop Ruby 2.5 due to EOL (#894)
+
+### 5.0.4
+- test: out_elasticsearch: Remove a needless headers from affinity stub (#888)
+- Target Index Affinity (#883)
+
+### 5.0.3
+- Fix use_legacy_template documentation (#880)
+- Add FAQ for dynamic index/template (#878)
+- Handle IPv6 address string on host and hosts parameters (#877)
+
 ### 5.0.2
 - GitHub Actions: Tweak Ruby versions on test (#875)
 - test: datastreams: Set nonexistent datastream as default (#874)

data/README.Troubleshooting.md

@@ -10,6 +10,7 @@
   + [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
   + [Fluentd seems to hang if it unable to connect Elasticsearch, why?](#fluentd-seems-to-hang-if-it-unable-to-connect-elasticsearch-why)
   + [Enable Index Lifecycle Management](#enable-index-lifecycle-management)
+    + [Configuring for dynamic index or template](#configuring-for-dynamic-index-or-template)
   + [How to specify index codec](#how-to-specify-index-codec)
   + [Cannot push logs to Elasticsearch with connect_write timeout reached, why?](#cannot-push-logs-to-elasticsearch-with-connect_write-timeout-reached-why)
 
@@ -524,6 +525,96 @@ template_name your-fluentd-template
 template_file /path/to/fluentd-template.json
 ```
 
+#### Configuring for dynamic index or template
+
+Some users want to setup ILM for dynamic index/template.
+`index_petterns` and `template.settings.index.lifecycle.name` in Elasticsearch template will be overwritten by Elasticsearch plugin:
+
+```json
+{
+  "index_patterns": ["mock"],
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "mock",
+          "rollover_alias": "mock"
+        },
+        "number_of_shards": "<<shard>>",
+        "number_of_replicas": "<<replica>>"
+      }
+    }
+  }
+}
+```
+
+This template will be handled with:
+
+```aconf
+<source>
+  @type http
+  port 5004
+  bind 0.0.0.0
+  body_size_limit 32m
+  keepalive_timeout 10s
+  <parse>
+    @type json
+  </parse>
+</source>
+
+<match kubernetes.var.log.containers.**etl-webserver**.log>
+    @type elasticsearch
+    @id out_es_etl_webserver
+    @log_level info
+    include_tag_key true
+    host $HOST
+    port $PORT
+    path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}"
+    request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '30s'}"
+    scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
+    ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
+    ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}"
+    reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}"
+    reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}"
+    reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}"
+    log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}"
+    logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'etl-webserver'}"
+    logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'false'}"
+    index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'etl-webserver'}"
+    type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}"
+    time_key "#{ENV['FLUENT_ELASTICSEARCH_TIME_KEY'] || '@timestamp'}"
+    include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'true'}"
+
+    # ILM Settings - WITH ROLLOVER support
+    # https://github.com/uken/fluent-plugin-elasticsearch#enable-index-lifecycle-management
+    application_name "etl-webserver"
+    index_date_pattern ""
+    # Policy configurations
+    enable_ilm true
+    ilm_policy_id etl-webserver
+    ilm_policy_overwrite true
+    ilm_policy {"policy": {"phases": {"hot": {"min_age": "0ms","actions": {"rollover": {"max_age": "5m","max_size": "3gb"},"set_priority": {"priority": 100}}},"delete": {"min_age": "30d","actions": {"delete": {"delete_searchable_snapshot": true}}}}}}
+    use_legacy_template false
+    template_name etl-webserver
+    template_file /configs/index-template.json
+    template_overwrite true
+    customize_template {"<<shard>>": "3","<<replica>>": "0"}
+
+    <buffer>
+        flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
+        flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
+        chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '8MB'}"
+        total_limit_size "#{ENV['FLUENT_ELASTICSEARCH_TOTAL_LIMIT_SIZE'] || '450MB'}"
+        queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
+        retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '60s'}"
+        retry_forever false
+    </buffer>
+</match>
+```
+
+For more details, please refer the discussion:
+https://github.com/uken/fluent-plugin-elasticsearch/issues/867
+
 ### How to specify index codec
 
 Elasticsearch can handle compression methods for stored data such as LZ4 and best_compression.

data/README.md

@@ -11,7 +11,7 @@ Send your logs to Elasticsearch (and search them with Kibana maybe?)
 
 Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-elasticsearch-service](https://github.com/atomita/fluent-plugin-aws-elasticsearch-service)
 
-Current maintainers: @cosmo0920
+Current maintainers: [Hiroshi Hatake | @cosmo0920](https://github.com/cosmo0920), [Kentaro Hayashi | @kenhys](https://github.com/kenhys)
 
 * [Installation](#installation)
 * [Usage](#usage)
@@ -38,6 +38,7 @@ Current maintainers: @cosmo0920
   + [suppress_type_name](#suppress_type_name)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
+  + [target_index_affinity](#target_index_affinity)
   + [template_name](#template_name)
   + [template_file](#template_file)
   + [template_overwrite](#template_overwrite)
@@ -171,6 +172,24 @@ You can specify Elasticsearch host by this parameter.
 
 **Note:** Since v3.3.2, `host` parameter supports builtin placeholders. If you want to send events dynamically into different hosts at runtime with `elasticsearch_dynamic` output plugin, please consider to switch to use plain `elasticsearch` output plugin. In more detail for builtin placeholders, please refer to [Placeholders](#placeholders) section.
 
+To use IPv6 address on `host` parameter, you can use the following styles:
+
+#### string style
+
+To use string style, you must quote IPv6 address due to prevent to be interpreted as JSON:
+
+```
+host "[2404:7a80:d440:3000:192a:a292:bd7f:ca10]"
+```
+
+#### raw style
+
+You can also specify raw IPv6 address. This will be handled as `[specified IPv6 address]`:
+
+```
+host 2404:7a80:d440:3000:192a:a292:bd7f:ca10
+```
+
 ### port
 
 ```
@@ -237,6 +256,16 @@ hosts host1:port1,host2:port2,host3 # port3 is 9200
 
 **Note:** Up until v2.8.5, it was allowed to embed the username/password in the URL. However, this syntax is deprecated as of v2.8.6 because it was found to cause serious connection problems (See #394). Please migrate your settings to use the `user` and `password` field (described below) instead.
 
+#### IPv6 addresses
+
+When you want to specify IPv6 addresses, you must specify schema together:
+
+```
+hosts http://[2404:7a80:d440:3000:de:7311:6329:2e6c]:port1,http://[2404:7a80:d440:3000:de:7311:6329:1e6c]:port2,http://[2404:7a80:d440:3000:de:6311:6329:2e6c]:port3
+```
+
+If you don't specify hosts with schema together, Elasticsearch plugin complains Invalid URI for them.
+
 ### user, password, path, scheme, ssl_verify
 
 ```
@@ -426,6 +455,75 @@ and this record will be written to the specified index (`logstash-2014.12.19`) r
 
 Similar to `target_index_key` config, find the type name to write to in the record under this key (or nested record). If key not found in record - fallback to `type_name` (default "fluentd").
 
+### target_index_affinity
+
+Enable plugin to dynamically select logstash time based target index in update/upsert operations based on already indexed data rather than current time of indexing.
+
+```
+target_index_affinity true # defaults to false
+```
+
+By default plugin writes data of logstash format index based on current time. For example daily based index after mignight data is written to newly created index. This is normally ok when data is coming from single source and not updated after indexing.
+
+But if you have a use case where data is also updated after indexing and `id_key` is used to identify the document uniquely for updating. Logstash format is wanted to be used for easy data managing and retention. Updates are done right after indexing to complete the data (all data not available from single source) and no updates are done anymore later point on time. In this case problem happends at index rotation time where write to 2 indexes with same id_key value may happen.
+
+This setting will search existing data by using elastic search's [id query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-ids-query.html) using `id_key` value (with logstash_prefix and logstash_prefix_separator index pattarn e.g. `logstash-*`). The index of found data is used for update/upsert. When no data is found, data is written to current logstash index as normally.
+
+This setting requires following other settings:
+```
+logstash_format true
+id_key myId  # Some field on your data to identify the data uniquely
+write_operation upsert  # upsert or update
+```
+
+Suppose you have the following situation where you have 2 different match to consume data from 2 different Kafka topics independently but close in time with each other (order not known).
+
+```
+  <match data1>
+    @type elasticsearch
+    ...
+    id_key myId
+    write_operation upsert
+    logstash_format true
+    logstash_dateformat %Y.%m.%d
+    logstash_prefix myindexprefix
+    target_index_affinity true
+    ...
+
+  <match data2>
+    @type elasticsearch
+    ...
+    id_key myId
+    write_operation upsert
+    logstash_format true
+    logstash_dateformat %Y.%m.%d
+    logstash_prefix myindexprefix
+    target_index_affinity true
+    ...
+```
+
+If your first (data1) input is:
+```
+{
+  "myId": "myuniqueId1",
+  "datafield1": "some value",
+}
+```
+
+and your second (data2) input is:
+```
+{
+  "myId": "myuniqueId1",
+  "datafield99": "some important data from other source tightly related to id myuniqueId1 and wanted to be in same document.",
+}
+```
+
+Date today is 10.05.2021 so data is written to index `myindexprefix-2021.05.10` when both data1 and data2 is consumed during today.
+But when we are close to index rotation and data1 is consumed and indexed at `2021-05-10T23:59:55.59707672Z` and data2
+is consumed a bit later at `2021-05-11T00:00:58.222079Z` i.e. logstash index has been rotated and normally data2 would have been written
+to index `myindexprefix-2021.05.11`. But with target_index_affinity setting as value true, data2 is now written to index `myindexprefix-2021.05.10`
+into same document with data1 as wanted and duplicated document is avoided.
+
 ### template_name
 
 The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
@@ -1325,9 +1423,9 @@ Default value is `nil`.
 
 Use legacy template or not.
 
-Elasticsearch 7.8 or later supports the brand new composable templates.
+For Elasticsearch 7.8 or later, users can specify this parameter as `false` if their [template_file](#template_file) contains a composable index template.
 
-For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
+For Elasticsearch 7.7 or older, users should specify this parameter as `true`.
 
 Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.0.2'
+  s.version       = '5.1.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -23,6 +23,10 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     unrecoverable_error_types.include?(type)
   end
 
+  def unrecoverable_record_error?(type)
+    ['json_parse_exception'].include?(type)
+  end
+
   def log_es_400_reason(&block)
     if @plugin.log_es_400_reason
       block.call
@@ -43,15 +47,17 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     stats = Hash.new(0)
     meta = {}
     header = {}
+    affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
     chunk.msgpack_each do |time, rawrecord|
       bulk_message = ''
       next unless rawrecord.is_a? Hash
       begin
         # we need a deep copy for process_message to alter
         processrecord = Marshal.load(Marshal.dump(rawrecord))
-        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
         next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
       rescue => e
+        @plugin.log.debug("Exception in error handler during deep copy: #{e}")
         stats[:bad_chunk_record] += 1
         next
       end
@@ -105,10 +111,15 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
           stats[type] += 1
-          retry_stream.add(time, rawrecord)
           if unrecoverable_error?(type)
             raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
           end
+          if unrecoverable_record_error?(type)
+            @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - #{type}: #{reason}"))
+            next
+          else
+            retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
+          end
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -32,13 +32,25 @@ module Fluent::ElasticsearchIndexTemplate
     return false
   end
 
+  def host_unreachable_exceptions
+    if Gem::Version.new(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
+      # elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
+      # Elasticsearch class on Transport.
+      # This is why #host_unreachable_exceptions is not callable directly
+      # via transport (not transport's transport instance accessor) any more.
+      client.transport.transport.host_unreachable_exceptions
+    else
+      client.transport.host_unreachable_exceptions
+    end
+  end
+
   def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
     transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
-    rescue *client.transport.host_unreachable_exceptions, *transport_errors, Timeout::Error => e
+    rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -2,6 +2,7 @@
 require 'date'
 require 'excon'
 require 'elasticsearch'
+require 'set'
 begin
   require 'elasticsearch/xpack'
 rescue LoadError
@@ -13,6 +14,7 @@ begin
   require 'strptime'
 rescue LoadError
 end
+require 'resolv'
 
 require 'fluent/plugin/output'
 require 'fluent/event'
@@ -70,7 +72,7 @@ module Fluent::Plugin
     DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
     DEFAULT_TYPE_NAME = "fluentd".freeze
     DEFAULT_RELOAD_AFTER = -1
-    TARGET_BULK_BYTES = 20 * 1024 * 1024
+    DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
 
     config_param :host, :string,  :default => 'localhost'
@@ -164,7 +166,7 @@ EOC
     config_param :suppress_doc_wrap, :bool, :default => false
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
-    config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
+    config_param :bulk_message_request_threshold, :size, :default => DEFAULT_TARGET_BULK_BYTES
     config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
@@ -174,6 +176,7 @@ EOC
     config_param :truncate_caches_interval, :time, :default => nil
     config_param :use_legacy_template, :bool, :default => true
     config_param :catch_transport_exception_on_retry, :bool, :default => true
+    config_param :target_index_affinity, :bool, :default => false
 
     config_section :metadata, param_name: :metainfo, multi: false do
       config_param :include_chunk_id, :bool, :default => false
@@ -668,7 +671,11 @@ EOC
           end
         end.compact
       else
-        [{host: @host, port: @port, scheme: @scheme.to_s}]
+        if Resolv::IPv6::Regex.match(@host)
+          [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
+        else
+          [{host: @host, port: @port, scheme: @scheme.to_s}]
+        end
       end.each do |host|
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path
@@ -829,13 +836,14 @@ EOC
                extract_placeholders(@host, chunk)
              end
 
+      affinity_target_indices = get_affinity_target_indices(chunk)
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
         record = inject_chunk_id_to_record_if_needed(record, chunk_id)
 
         begin
-          meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
+          meta, header, record = process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
           info = if @include_index_in_url
                    RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
                  else
@@ -872,6 +880,42 @@ EOC
       end
     end
 
+    def target_index_affinity_enabled?()
+      @target_index_affinity && @logstash_format && @id_key && (@write_operation == UPDATE_OP || @write_operation == UPSERT_OP)
+    end
+
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      if target_index_affinity_enabled?()
+        id_key_accessor = record_accessor_create(@id_key)
+        ids = Set.new
+        chunk.msgpack_each do |time, record|
+          next unless record.is_a? Hash
+          begin
+            ids << id_key_accessor.call(record)
+          end
+        end
+        log.debug("Find affinity target_indices by quering on ES (write_operation #{@write_operation}) for ids: #{ids.to_a}")
+        options = {
+          :index => "#{logstash_prefix}#{@logstash_prefix_separator}*",
+        }
+        query = {
+          'query' => { 'ids' => { 'values' => ids.to_a } },
+          '_source' => false,
+          'sort' => [
+            {"_index" => {"order" => "desc"}}
+         ]
+        }
+        result = client.search(options.merge(:body => Yajl.dump(query)))
+        # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
+        result['hits']['hits'].each do |hit|
+          indices[hit["_id"]] = hit["_index"]
+          log.debug("target_index for id: #{hit["_id"]} from es: #{hit["_index"]}")
+        end
+      end
+      indices
+    end
+
     def split_request?(bulk_message, info)
       # For safety.
     end
@@ -884,7 +928,7 @@ EOC
       false
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
 
       if @flatten_hashes
@@ -925,6 +969,15 @@ EOC
         record[@tag_key] = tag
       end
 
+      # If affinity target indices map has value for this particular id, use it as target_index
+      if !affinity_target_indices.empty?
+        id_accessor = record_accessor_create(@id_key)
+        id_value = id_accessor.call(record)
+        if affinity_target_indices.key?(id_value)
+          target_index = affinity_target_indices[id_value]
+        end
+      end
+
       target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
       if target_type_parent && target_type_parent[target_type_child_key]
         target_type = target_type_parent.delete(target_type_child_key)