fluent-plugin-elasticsearch 1.11.1 → 1.12.0.rc.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 101b819a34a206c64e8829e72793158b8f5e448e
-  data.tar.gz: ffc95b8b3649186630a722e15dbe77f70ce7efa6
+  metadata.gz: 7214256a17bb6711403a1eee956cb8c3a040b8d4
+  data.tar.gz: cd8677126edbf697885826f46fa92c0519821f18
 SHA512:
-  metadata.gz: caf081c1c9dcbf623bba5e649333b8e6790d9700c2931c0964cf8a0711f346d45355454a2d2e675d0e415252e77d76311a997d6e1cfbe64ab5290644c2d85dd6
-  data.tar.gz: e137b0dc4de293528fe907f60e06007ff51ac860ea0cf040974a246f799ea7e6e582a49ba6c2f804b0dacf2ad4784d9ba0a387185d4010cbc1c469990ceb1583
+  metadata.gz: dcd8ef4cf0ddd8ab0eb6ca11020ee97ec823727c53fe5eb56a0460c9588e4b1ac13a99bc996bae44872a1cfa6a3bc88f68ad391efacdedad08d6092522d69857
+  data.tar.gz: b8aeeda2d04b325dca968b38dacf3f150d1a1239c431bcb6f0c1b3becbbf59b5922dd6289e35eb5e4885944ae3b940acad5f271c873c3ad9948e644f02a831de

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 1.12.0.rc.1
+- Backport separating generate hash id module and bundled new plugin for generating unique hash id (#331)
+
 ### 1.11.1
 - Raise ConfigError when specifying different @hash_config.hash_id_key and id_key configration (#326)
 - backport small typo fix in README.md (#328)

data/README.md

@@ -517,14 +517,20 @@ Note that the flattener does not deal with arrays at this time.
 
 By default, the fluentd elasticsearch plugin does not emit records with a _id field, leaving it to Elasticsearch to generate a unique _id as the record is indexed. When an Elasticsearch cluster is congested and begins to take longer to respond than the configured request_timeout, the fluentd elasticsearch plugin will re-send the same bulk request. Since Elasticsearch can't tell its actually the same request, all documents in the request are indexed again resulting in duplicate data. In certain scenarios, this can result in essentially and infinite loop generating multiple copies of the same data.
 
-Using an _id generated by the fluentd elasticsearch plugin will communicate to Elasticsearch the uniqueness of the requests so that duplicates will be rejected or simply replace the existing records.
+The bundled elasticsearch_genid filter can generate a unique _hash key for each record, this key may be passed to the id_key parameter in the elasticsearch plugin to communicate to Elasticsearch the uniqueness of the requests so that duplicates will be rejected or simply replace the existing records.
 Here is a sample config:
 
 ```
-<hash>
-  hash_id_key _hash    # storing generated hash id key
-</hash>
-id_key _hash # specify same key name which is specified in hash_id_key
+<filter>
+  @type elasticsearch_genid
+  hash_id_key _hash    # storing generated hash id key (default is _hash)
+</filter>
+<match **>
+  @type elasticsearch
+  id_key _hash # specify same key name which is specified in hash_id_key
+  remove_keys _hash # Elasticsearch doesn't like keys that start with _
+  # other settings are ommitted.
+</match>
 ```
 
 ### Not seeing a config you need?

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.11.1'
+  s.version       = '1.12.0.rc.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -0,0 +1,25 @@
+require 'securerandom'
+require 'base64'
+require 'fluent/filter'
+
+module Fluent
+  class ElasticsearchGenidFilter < Filter
+    Fluent::Plugin.register_filter('elasticsearch_genid', self)
+
+    config_param :hash_id_key, :string, :default => '_hash'
+
+    def initialize
+      super
+    end
+
+    def configure(conf)
+      super
+    end
+
+    def filter(tag, time, record)
+      record[@hash_id_key] = Base64.strict_encode64(SecureRandom.uuid)
+      record
+    end
+
+  end
+end

data/lib/fluent/plugin/generate_hash_id_support.rb

@@ -6,7 +6,9 @@ module Fluent
     def self.included(klass)
       klass.instance_eval {
         config_section :hash, param_name: :hash_config, required: false, multi: false do
-          config_param :hash_id_key, :string, default: '_hash'
+          config_param :hash_id_key, :string, default: '_hash',
+                       obsoleted: "Use bundled filer-elasticsearch-genid instead."
+
         end
       }
     end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -242,14 +242,6 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     end.join(', ')
   end
 
-  BODY_DELIMITER = "\n".freeze
-  UPDATE_OP = "update".freeze
-  UPSERT_OP = "upsert".freeze
-  CREATE_OP = "create".freeze
-  INDEX_OP = "index".freeze
-  ID_FIELD = "_id".freeze
-  TIMESTAMP_FIELD = "@timestamp".freeze
-
   def append_record_to_messages(op, meta, header, record, msgs)
     case op
     when UPDATE_OP, UPSERT_OP

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -0,0 +1,40 @@
+require 'helper'
+require 'date'
+require 'json'
+require 'flexmock/test_unit'
+require 'fluent/plugin/filter_elasticsearch_genid'
+
+class ElasticsearchGenidFilterTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf='')
+    Fluent::Test::FilterTestDriver.new(Fluent::ElasticsearchGenidFilter).configure(conf)
+  end
+
+  def sample_record
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal '_hash', d.instance.hash_id_key
+  end
+
+  data("default" => {"hash_id_key" => "_hash"},
+       "custom_key" => {"hash_id_key" => "_edited"},
+      )
+  def test_filter(data)
+    d = create_driver("hash_id_key #{data["hash_id_key"]}")
+    flexmock(SecureRandom).should_receive(:uuid)
+      .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
+    d.run do
+      d.filter(sample_record)
+    end
+    assert_equal(Base64.strict_encode64(SecureRandom.uuid),
+                 d.filtered_as_array.map {|e| e.last}.first[d.instance.hash_id_key])
+  end
+end

data/test/plugin/test_out_elasticsearch.rb

@@ -205,7 +205,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_configure_with_invaild_generate_id_config
-    assert_raise(Fluent::ConfigError) {
+    assert_raise_message(/Use bundled filer-elasticsearch-genid instead./) do
       driver.configure(Fluent::Config::Element.new(
                          'ROOT', '', {
                            '@type' => 'elasticsearch',
@@ -216,7 +216,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                                                        }, [])
                          ]
                        ))
-    }
+    end
   end
 
   def test_template_already_present
@@ -558,25 +558,19 @@ class ElasticsearchOutput < Test::Unit::TestCase
          "custom hash_id_key" => {"hash_id_key" => '_hash_id'},
         )
     def test_writes_with_genrate_hash(data)
-      driver.configure(Fluent::Config::Element.new(
-                         'ROOT', '', {
-                           '@type' => 'elasticsearch',
-                           'id_key' => data["hash_id_key"],
-                         }, [
-                           Fluent::Config::Element.new('hash', '', {
-                                                         'keys' => ['request_id'],
-                                                         'hash_id_key' => data["hash_id_key"],
-                                                       }, [])
-                         ]
-                       ))
-      stub_elastic_ping
-      stub_elastic
-      flexmock(SecureRandom).should_receive(:uuid)
-        .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
-      time = Time.parse("2017-10-15 15:00:23.34567890 UTC").to_i
-      driver.emit(sample_record.merge('request_id' => 'elastic'), time)
-      driver.run
-      assert_equal(Base64.strict_encode64(SecureRandom.uuid), index_cmds[1]["#{data["hash_id_key"]}"])
+      assert_raise_message(/Use bundled filer-elasticsearch-genid instead./) do
+        driver.configure(Fluent::Config::Element.new(
+                           'ROOT', '', {
+                             '@type' => 'elasticsearch',
+                             'id_key' => data["hash_id_key"],
+                           }, [
+                             Fluent::Config::Element.new('hash', '', {
+                                                           'keys' => ['request_id'],
+                                                           'hash_id_key' => data["hash_id_key"],
+                                                         }, [])
+                           ]
+                         ))
+      end
     end
   end
 

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -313,26 +313,19 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
          "custom hash_id_key" => {"hash_id_key" => '_hash_id'},
         )
     def test_writes_with_genrate_hash(data)
-      driver.configure(Fluent::Config::Element.new(
-                         'ROOT', '', {
-                           '@type' => 'elasticsearch',
-                           'id_key' => data["hash_id_key"],
-                         }, [
-                           Fluent::Config::Element.new('hash', '', {
-                                                         'keys' => ['request_id'],
-                                                         'hash_id_key' => data["hash_id_key"],
-                                                       }, [])
-                         ]
-                       ))
-      stub_elastic_ping
-      stub_elastic
-      stub_elastic
-      flexmock(SecureRandom).should_receive(:uuid)
-        .and_return("82120f33-897a-4d9d-b3d5-14afd18fb412")
-      time = Time.parse("2017-10-15 15:00:23.34567890 UTC").to_i
-      driver.emit(sample_record.merge('request_id' => 'elastic'), time)
-      driver.run
-      assert_equal(Base64.strict_encode64(SecureRandom.uuid), index_cmds[1]["#{data["hash_id_key"]}"])
+      assert_raise_message(/Use bundled filer-elasticsearch-genid instead./) do
+        driver.configure(Fluent::Config::Element.new(
+                           'ROOT', '', {
+                             '@type' => 'elasticsearch',
+                             'id_key' => data["hash_id_key"],
+                           }, [
+                             Fluent::Config::Element.new('hash', '', {
+                                                           'keys' => ['request_id'],
+                                                           'hash_id_key' => data["hash_id_key"],
+                                                         }, [])
+                           ]
+                         ))
+      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.11.1
+  version: 1.12.0.rc.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-21 00:00:00.000000000 Z
+date: 2017-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -147,10 +147,12 @@ files:
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
+- lib/fluent/plugin/filter_elasticsearch_genid.rb
 - lib/fluent/plugin/generate_hash_id_support.rb
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
+- test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json
@@ -169,9 +171,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13
@@ -180,6 +182,7 @@ specification_version: 4
 summary: ElasticSearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json