fluent-plugin-elasticsearch 1.10.2 → 1.10.3.rc.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7756405c1e59422b26624fbd6e3a0fded30c6d46
-  data.tar.gz: c12bd57fb39982e51e9a5a4dfaa73f3ef5da3e2c
+  metadata.gz: 91687e424c6bff1383a85347f0cf9d6aecd76631
+  data.tar.gz: f0a943997d76aa6a63234915ac44868d0f2285a2
 SHA512:
-  metadata.gz: 7fb789e36c7a2ad8ae823fa4ed0f7578293bd70426629fccdb09681cc82bbc4e7183439dee046a117bb08ea7de4bcc69eed8052822e5c454e1934218db8d23f2
-  data.tar.gz: ea0983374a67d7f23ad2e5c2a043de495131784b6b583abbd12d483946dc13aa9fc01e9601fabded84ef11404efd23643f280ff9623aafdb4ce5d1a02f8db43a
+  metadata.gz: a9ceabd8ac3832d51174b955c08f24ce683f95411b1b97afc3565018dca8ddde55d679ae84099815a71bb86691dafa03cbcc3405879975fa6684badc31de6b0b
+  data.tar.gz: d2663e6accd44050cc16eb8a85a5f521f399ca503a8fb3e866715b918f14db4dbf0df175db6ae1c7dc3f3695f19972f27a6f1f79ed2944c83464d4c522136e08

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 1.10.3.rc.1
+- backport Add generating hash id mechanism to avoid records duplication (#323)
+
 ### 1.10.2
 - backport adding `include_timestamp` option (#311)
 

data/README.md

@@ -51,6 +51,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [Proxy Support](#proxy-support)
   + [Buffered output options](#buffered-output-options)
   + [Hash flattening](#hash-flattening)
+  + [Generate Hash ID](#generate-hash-id)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
 * [Contact](#contact)
@@ -332,7 +333,7 @@ reload_on_failure true # defaults to false
 You can set in the elasticsearch-transport how often dead connections from the elasticsearch-transport's pool will be resurrected.
 
 ```
-resurrect_after 5 # defaults to 60s
+resurrect_after 5s # defaults to 60s
 ```
 
 ### include_tag_key, tag_key
@@ -485,7 +486,7 @@ Starting with version 0.8.0, this gem uses excon, which supports proxy with envi
 
 ```
 buffer_type memory
-flush_interval 60
+flush_interval 60s
 retry_limit 17
 retry_wait 1.0
 num_threads 1
@@ -512,6 +513,19 @@ This will produce elasticsearch output that looks like this:
 
 Note that the flattener does not deal with arrays at this time.
 
+### Generate Hash ID
+
+By default, the fluentd elasticsearch plugin does not emit records with a _id field, leaving it to Elasticsearch to generate a unique _id as the record is indexed. When an Elasticsearch cluster is congested and begins to take longer to respond than the configured request_timeout, the fluentd elasticsearch plugin will re-send the same bulk request. Since Elasticsearch can't tell its actually the same request, all documents in the request are indexed again resulting in duplicate data. In certain scenarios, this can result in essentially and infinite loop generating multiple copies of the same data.
+
+Using an _id generated by the fluentd elasticsearch plugin will communicate to Elasticsearch the uniqueness of the requests so that duplicates will be rejected or simply replace the existing records.
+Here is a sample config:
+
+```
+<hash>
+  hash_id_key _id    # storing generated hash id key
+<hash>
+```
+
 ### Not seeing a config you need?
 
 We try to keep the scope of this plugin small and not add too many configuration options. If you think an option would be useful to others, feel free to open an issue or contribute a Pull Request.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.10.2'
+  s.version       = '1.10.3.rc.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/generate_hash_id_support.rb

@@ -0,0 +1,21 @@
+require 'securerandom'
+require 'base64'
+
+module Fluent
+  module GenerateHashIdSupport
+    def self.included(klass)
+      klass.instance_eval {
+        config_section :hash, param_name: :hash_config, required: false, multi: false do
+          config_param :hash_id_key, :string, default: '_id'
+        end
+      }
+    end
+
+    def generate_hash_id_key(record)
+      s = ""
+      s += Base64.strict_encode64(SecureRandom.uuid)
+      record[@hash_config.hash_id_key] = s
+      record
+    end
+  end
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -11,6 +11,7 @@ end
 
 require 'fluent/output'
 require_relative 'elasticsearch_index_template'
+require_relative 'generate_hash_id_support'
 
 class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   class ConnectionFailure < StandardError; end
@@ -67,6 +68,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :pipeline, :string, :default => nil
 
   include Fluent::ElasticsearchIndexTemplate
+  include Fluent::GenerateHashIdSupport
 
   def initialize
     super
@@ -311,6 +313,10 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
         record = flatten_record(record)
       end
 
+      if @hash_config
+        record = generate_hash_id_key(record)
+      end
+
       dt = nil
       if @logstash_format || @include_timestamp
         if record.has_key?(TIMESTAMP_FIELD)

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -10,6 +10,8 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
   DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
   DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
 
+  include Fluent::GenerateHashIdSupport
+
   attr_reader :dynamic_config
 
   def configure(conf)
@@ -121,6 +123,10 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
     chunk.msgpack_each do |time, record|
       next unless record.is_a? Hash
 
+      if @hash_config
+        record = generate_hash_id_key(record)
+      end
+
       begin
         # evaluate all configurations here
         DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|

data/test/plugin/test_out_elasticsearch.rb

@@ -1,7 +1,10 @@
 require 'helper'
 require 'date'
+require 'flexmock/test_unit'
 
 class ElasticsearchOutput < Test::Unit::TestCase
+  include FlexMock::TestCase
+
   attr_accessor :index_cmds, :index_command_counts
 
   def setup
@@ -410,6 +413,33 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
+  class AdditionalHashIdMechanismTest < self
+    data("default"            => {"hash_id_key" => '_id'},
+         "custom hash_id_key" => {"hash_id_key" => '_hash_id'},
+        )
+    def test_writes_with_genrate_hash(data)
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'id_key' => data["hash_id_key"],
+                         }, [
+                           Fluent::Config::Element.new('hash', '', {
+                                                         'keys' => ['request_id'],
+                                                         'hash_id_key' => data["hash_id_key"],
+                                                       }, [])
+                         ]
+                       ))
+      stub_elastic_ping
+      stub_elastic
+      flexmock(SecureRandom).should_receive(:uuid)
+        .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
+      time = Time.parse("2017-10-15 15:00:23.34567890 UTC").to_i
+      driver.emit(sample_record.merge('request_id' => 'elastic'), time)
+      driver.run
+      assert_equal(Base64.strict_encode64(SecureRandom.uuid), index_cmds[1]["#{data["hash_id_key"]}"])
+    end
+  end
+
   def test_writes_to_speficied_index_uppercase
     driver.configure("index_name MyIndex\n")
     stub_elastic_ping

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -308,6 +308,34 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(2000, total)
   end
 
+  class AdditionalHashIdMechanismTest < self
+    data("default"            => {"hash_id_key" => '_id'},
+         "custom hash_id_key" => {"hash_id_key" => '_hash_id'},
+        )
+    def test_writes_with_genrate_hash(data)
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'id_key' => data["hash_id_key"],
+                         }, [
+                           Fluent::Config::Element.new('hash', '', {
+                                                         'keys' => ['request_id'],
+                                                         'hash_id_key' => data["hash_id_key"],
+                                                       }, [])
+                         ]
+                       ))
+      stub_elastic_ping
+      stub_elastic
+      stub_elastic
+      flexmock(SecureRandom).should_receive(:uuid)
+        .and_return("82120f33-897a-4d9d-b3d5-14afd18fb412")
+      time = Time.parse("2017-10-15 15:00:23.34567890 UTC").to_i
+      driver.emit(sample_record.merge('request_id' => 'elastic'), time)
+      driver.run
+      assert_equal(Base64.strict_encode64(SecureRandom.uuid), index_cmds[1]["#{data["hash_id_key"]}"])
+    end
+  end
+
   def test_makes_bulk_request
     stub_elastic_ping
     stub_elastic

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.10.2
+  version: 1.10.3.rc.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-25 00:00:00.000000000 Z
+date: 2017-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -145,6 +145,7 @@ files:
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec
 - lib/fluent/plugin/elasticsearch_index_template.rb
+- lib/fluent/plugin/generate_hash_id_support.rb
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
@@ -166,9 +167,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13