fluent-plugin-elasticsearch 5.0.3 → 5.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6eb418d889b91bf79c37c1cd72789981eb4133bcfc34e21b26e79bb919462272
-  data.tar.gz: 168ddc77fb73216da63f1ce65ed673cf12ab13d6db549fc3e2da81956c938990
+  metadata.gz: 7993521144deb5ebe2665d231cdeda9833cb4b3ba87909a2e21b96f8a43f61cc
+  data.tar.gz: 25e0341fe8d2b131350fb521c520fa31314cdcf478914aaad2d4a3bdbf5a3954
 SHA512:
-  metadata.gz: 5c4a2a8f63b25ea8785e0d4ef291043a93a86fea8f42638caaf46dd3efe29fd0a12919dd339d8a28c97aef65494ba054a3ff3312a122ab2cd89cb54cb069055c
-  data.tar.gz: 6b13851ce29b2a6f2083ebc57ae733a9ca00d79a9fd09ac4c7630d99a9c6d2da302c48f2e0cd901810e0a851911e14f8355fc336e21afd97d3f33fe205b0c78a
+  metadata.gz: db5b6b3f9f4fc1e1d6a9c438e371aa73f962798f8a39519a01bfdeb38bfcb7a8b8e6b4efe20202108d518c33dd0701c29c20c7d4b7c0bb01b16f5907f34dea5d
+  data.tar.gz: 8c69a02d9cda795457f104177773939eb53e652a74f4ebeab4d8dc2b6f943222f9ca967eeca93b0ca283f8be4fa5d381b1300f762d6b5d7e77a07bebb4194524

data/History.md

@@ -1,6 +1,12 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+
+### 5.0.4
+- test: out_elasticsearch: Remove a needless headers from affinity stub (#888)
+- Target Index Affinity (#883)
+
+### 5.0.3
 - Fix use_legacy_template documentation (#880)
 - Add FAQ for dynamic index/template (#878)
 - Handle IPv6 address string on host and hosts parameters (#877)

data/README.md

@@ -38,6 +38,7 @@ Current maintainers: @cosmo0920
   + [suppress_type_name](#suppress_type_name)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
+  + [target_index_affinity](#target_index_affinity)
   + [template_name](#template_name)
   + [template_file](#template_file)
   + [template_overwrite](#template_overwrite)
@@ -454,6 +455,75 @@ and this record will be written to the specified index (`logstash-2014.12.19`) r
 
 Similar to `target_index_key` config, find the type name to write to in the record under this key (or nested record). If key not found in record - fallback to `type_name` (default "fluentd").
 
+### target_index_affinity
+
+Enable plugin to dynamically select logstash time based target index in update/upsert operations based on already indexed data rather than current time of indexing.
+
+```
+target_index_affinity true # defaults to false
+```
+
+By default plugin writes data of logstash format index based on current time. For example daily based index after mignight data is written to newly created index. This is normally ok when data is coming from single source and not updated after indexing.
+
+But if you have a use case where data is also updated after indexing and `id_key` is used to identify the document uniquely for updating. Logstash format is wanted to be used for easy data managing and retention. Updates are done right after indexing to complete the data (all data not available from single source) and no updates are done anymore later point on time. In this case problem happends at index rotation time where write to 2 indexes with same id_key value may happen.
+
+This setting will search existing data by using elastic search's [id query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-ids-query.html) using `id_key` value (with logstash_prefix and logstash_prefix_separator index pattarn e.g. `logstash-*`). The index of found data is used for update/upsert. When no data is found, data is written to current logstash index as normally.
+
+This setting requires following other settings:
+```
+logstash_format true
+id_key myId  # Some field on your data to identify the data uniquely
+write_operation upsert  # upsert or update
+```
+
+Suppose you have the following situation where you have 2 different match to consume data from 2 different Kafka topics independently but close in time with each other (order not known).
+
+```
+  <match data1>
+    @type elasticsearch
+    ...
+    id_key myId
+    write_operation upsert
+    logstash_format true
+    logstash_dateformat %Y.%m.%d
+    logstash_prefix myindexprefix
+    target_index_affinity true
+    ...
+
+  <match data2>
+    @type elasticsearch
+    ...
+    id_key myId
+    write_operation upsert
+    logstash_format true
+    logstash_dateformat %Y.%m.%d
+    logstash_prefix myindexprefix
+    target_index_affinity true
+    ...
+```
+
+If your first (data1) input is:
+```
+{
+  "myId": "myuniqueId1",
+  "datafield1": "some value",
+}
+```
+
+and your second (data2) input is:
+```
+{
+  "myId": "myuniqueId1",
+  "datafield99": "some important data from other source tightly related to id myuniqueId1 and wanted to be in same document.",
+}
+```
+
+Date today is 10.05.2021 so data is written to index `myindexprefix-2021.05.10` when both data1 and data2 is consumed during today.
+But when we are close to index rotation and data1 is consumed and indexed at `2021-05-10T23:59:55.59707672Z` and data2
+is consumed a bit later at `2021-05-11T00:00:58.222079Z` i.e. logstash index has been rotated and normally data2 would have been written
+to index `myindexprefix-2021.05.11`. But with target_index_affinity setting as value true, data2 is now written to index `myindexprefix-2021.05.10`
+into same document with data1 as wanted and duplicated document is avoided.
+
 ### template_name
 
 The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.0.3'
+  s.version       = '5.0.4'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -43,13 +43,14 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     stats = Hash.new(0)
     meta = {}
     header = {}
+    affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
     chunk.msgpack_each do |time, rawrecord|
       bulk_message = ''
       next unless rawrecord.is_a? Hash
       begin
         # we need a deep copy for process_message to alter
         processrecord = Marshal.load(Marshal.dump(rawrecord))
-        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
         next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
       rescue => e
         stats[:bad_chunk_record] += 1

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -2,6 +2,7 @@
 require 'date'
 require 'excon'
 require 'elasticsearch'
+require 'set'
 begin
   require 'elasticsearch/xpack'
 rescue LoadError
@@ -175,6 +176,7 @@ EOC
     config_param :truncate_caches_interval, :time, :default => nil
     config_param :use_legacy_template, :bool, :default => true
     config_param :catch_transport_exception_on_retry, :bool, :default => true
+    config_param :target_index_affinity, :bool, :default => false
 
     config_section :metadata, param_name: :metainfo, multi: false do
       config_param :include_chunk_id, :bool, :default => false
@@ -834,13 +836,14 @@ EOC
                extract_placeholders(@host, chunk)
              end
 
+      affinity_target_indices = get_affinity_target_indices(chunk)
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
         record = inject_chunk_id_to_record_if_needed(record, chunk_id)
 
         begin
-          meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
+          meta, header, record = process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
           info = if @include_index_in_url
                    RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
                  else
@@ -877,6 +880,42 @@ EOC
       end
     end
 
+    def target_index_affinity_enabled?()
+      @target_index_affinity && @logstash_format && @id_key && (@write_operation == UPDATE_OP || @write_operation == UPSERT_OP)
+    end
+
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      if target_index_affinity_enabled?()
+        id_key_accessor = record_accessor_create(@id_key)
+        ids = Set.new
+        chunk.msgpack_each do |time, record|
+          next unless record.is_a? Hash
+          begin
+            ids << id_key_accessor.call(record)
+          end
+        end
+        log.debug("Find affinity target_indices by quering on ES (write_operation #{@write_operation}) for ids: #{ids.to_a}")
+        options = {
+          :index => "#{logstash_prefix}#{@logstash_prefix_separator}*",
+        }
+        query = {
+          'query' => { 'ids' => { 'values' => ids.to_a } },
+          '_source' => false,
+          'sort' => [
+            {"_index" => {"order" => "desc"}}
+         ]
+        }
+        result = client.search(options.merge(:body => Yajl.dump(query)))
+        # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
+        result['hits']['hits'].each do |hit|
+          indices[hit["_id"]] = hit["_index"]
+          log.debug("target_index for id: #{hit["_id"]} from es: #{hit["_index"]}")
+        end
+      end
+      indices
+    end
+
     def split_request?(bulk_message, info)
       # For safety.
     end
@@ -889,7 +928,7 @@ EOC
       false
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
 
       if @flatten_hashes
@@ -930,6 +969,15 @@ EOC
         record[@tag_key] = tag
       end
 
+      # If affinity target indices map has value for this particular id, use it as target_index
+      if !affinity_target_indices.empty?
+        id_accessor = record_accessor_create(@id_key)
+        id_value = id_accessor.call(record)
+        if affinity_target_indices.key?(id_value)
+          target_index = affinity_target_indices[id_value]
+        end
+      end
+
       target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
       if target_type_parent && target_type_parent[target_type_child_key]
         target_type = target_type_parent.delete(target_type_child_key)

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -27,10 +27,15 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
        @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       return [meta, header, record]
     end
 
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      indices
+    end
+
     def append_record_to_messages(op, meta, header, record, msgs)
       if record.has_key?('raise') && record['raise']
         raise Exception('process_message')

data/test/plugin/test_out_elasticsearch.rb

@@ -10,7 +10,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   include FlexMock::TestCase
   include Fluent::Test::Helpers
 
-  attr_accessor :index_cmds, :index_command_counts
+  attr_accessor :index_cmds, :index_command_counts, :index_cmds_all_requests
 
   def setup
     Fluent::Test.setup
@@ -70,6 +70,14 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
+  def stub_elastic_all_requests(url="http://localhost:9200/_bulk")
+    @index_cmds_all_requests = Array.new
+    stub_request(:post, url).with do |req|
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_cmds_all_requests << @index_cmds
+    end
+  end
+
   def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
     stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
   end
@@ -4094,6 +4102,185 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal(pipeline, index_cmds.first['index']['pipeline'])
   end
 
+  def stub_elastic_affinity_target_index_search_with_body(url="http://localhost:9200/logstash-*/_search", ids, return_body_str)
+    # Note: ids used in query is unique list of ids
+    stub_request(:post, url)
+      .with(
+        body: "{\"query\":{\"ids\":{\"values\":#{ids.uniq.to_json}}},\"_source\":false,\"sort\":[{\"_index\":{\"order\":\"desc\"}}]}",
+      )
+      .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => return_body_str
+      }
+    end)
+  end
+
+  def stub_elastic_affinity_target_index_search(url="http://localhost:9200/logstash-*/_search", ids, indices)
+    # Example ids and indices arrays.
+    #  [ "3408a2c8eecd4fbfb82e45012b54fa82", "2816fc6ef4524b3f8f7e869002005433"]
+    #  [ "logstash-2021.04.28", "logstash-2021.04.29"]
+    body = %({
+      "took" : 31,
+      "timed_out" : false,
+      "_shards" : {
+        "total" : 52,
+        "successful" : 52,
+        "skipped" : 48,
+        "failed" : 0
+      },
+      "hits" : {
+        "total" : {
+          "value" : 356,
+          "relation" : "eq"
+        },
+        "max_score" : null,
+        "hits" : [
+          {
+            "_index" : "#{indices[0]}",
+            "_type" : "_doc",
+            "_id" : "#{ids[0]}",
+            "_score" : null,
+            "sort" : [
+              "#{indices[0]}"
+            ]
+          },
+          {
+            "_index" : "#{indices[1]}",
+            "_type" : "_doc",
+            "_id" : "#{ids[1]}",
+            "_score" : null,
+            "sort" : [
+              "#{indices[1]}"
+            ]
+          }
+        ]
+      }
+    })
+    stub_elastic_affinity_target_index_search_with_body(ids, body)
+  end
+
+  def stub_elastic_affinity_target_index_search_return_empty(url="http://localhost:9200/logstash-*/_search", ids)
+    empty_body = %({
+      "took" : 5,
+      "timed_out" : false,
+      "_shards" : {
+        "total" : 54,
+        "successful" : 54,
+        "skipped" : 53,
+        "failed" : 0
+      },
+      "hits" : {
+        "total" : {
+          "value" : 0,
+          "relation" : "eq"
+        },
+        "max_score" : null,
+        "hits" : [ ]
+      }
+    })
+    stub_elastic_affinity_target_index_search_with_body(ids, empty_body)
+  end
+
+  def test_writes_to_affinity_target_index
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    indices = ["logstash-2021.04.28"]
+    stub_elastic
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_write_operation_upsert
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation upsert")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    indices = ["logstash-2021.04.28"]
+    stub_elastic
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_index_not_exists_yet
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    stub_elastic
+    stub_elastic_affinity_target_index_search_return_empty(ids)
+    time = Time.parse Date.today.iso8601
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record('my_id' => my_id_value))
+    end
+    assert_equal("logstash-#{time.utc.strftime("%Y.%m.%d")}", index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_multiple_indices
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "2816fc6ef4524b3f8f7e869002005433"
+    my_id_value2 = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value, my_id_value2]
+    indices = ["logstash-2021.04.29", "logstash-2021.04.28"]
+    stub_elastic_all_requests
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+      driver.feed(sample_record('my_id' => my_id_value2))
+    end
+    assert_equal(2, index_cmds_all_requests.count)
+    assert_equal('logstash-2021.04.29', index_cmds_all_requests[0].first['update']['_index'])
+    assert_equal(my_id_value, index_cmds_all_requests[0].first['update']['_id'])
+    assert_equal('logstash-2021.04.28', index_cmds_all_requests[1].first['update']['_index'])
+    assert_equal(my_id_value2, index_cmds_all_requests[1].first['update']['_id'])
+  end
+
+  def test_writes_to_affinity_target_index_same_id_dublicated_write_to_oldest_index
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "2816fc6ef4524b3f8f7e869002005433"
+    # It may happen than same id has inserted to two index while data inserted during rollover period
+    ids = [my_id_value, my_id_value]
+    # Simulate the used sorting here, as search sorts indices in DESC order to pick only oldest index per single _id
+    indices = ["logstash-2021.04.29", "logstash-2021.04.28"]
+
+    stub_elastic_all_requests
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+
+    assert_equal(1, index_cmds_all_requests.count)
+    assert_equal('logstash-2021.04.28', index_cmds_all_requests[0].first['update']['_index'])
+    assert_equal(my_id_value, index_cmds_all_requests[0].first['update']['_id'])
+  end
+
   class PipelinePlaceholdersTest < self
     def test_writes_to_default_index_with_pipeline_tag_placeholder
       pipeline = "fluentd-${tag}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.0.3
+  version: 5.0.4
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2021-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd