fluent-plugin-elasticsearch 2.1.1 → 2.2.0.rc.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97d1627c0193a5dce42c5154bc62c1609e18dd9e
-  data.tar.gz: e37519936763de220cc909f3bcb77b6cb97ce7b5
+  metadata.gz: 90e7f44770dc48c5061c88be01ef839bed6cb5bb
+  data.tar.gz: bbf38e0fb4f108affcb5d0f868deacf540d432fc
 SHA512:
-  metadata.gz: 918c75ef1df3f9bbbe40c83db330c22618fcddfadcdffd6d02a73a6dc647939899fc3d2f4a3c4b553e16e9cb6fe64d2814b43d064a85d89412a893eecca5f5f1
-  data.tar.gz: 13e526c7723c3db9889549a57c808591ee2793f981be0dcf0ffa375828bbe33ff59d790df60da8e38499b7d59885821c0672621cf51456030329bf5ab3f48b89
+  metadata.gz: 671b7654c928aa0d8f919266c6cbc9861ecf080e5da9e9529852bcc86322d62904d32bcc952658fa83d842bbb904eab1a31e14f0ef0191c4ad0a056307a305e5
+  data.tar.gz: e82257e8d545eec34e862e2198fabd936bb0622e64b58f78d6d8f1d3d6bba98ec7309e4444a50f574f8382189984b771fed9918af9ff2238d6324d97845f8e9c

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.2.0.rc.1
+- Separate generate hash id module and bundled new plugin for generating unique hash id (#331)
+
 ### 2.1.1
 - Raise ConfigError when specifying different @hash_config.hash_id_key and id_key configration (#327)
 - Small typo fix in README.md (#325)

data/README.md

@@ -557,15 +557,20 @@ Note that the flattener does not deal with arrays at this time.
 
 By default, the fluentd elasticsearch plugin does not emit records with a _id field, leaving it to Elasticsearch to generate a unique _id as the record is indexed. When an Elasticsearch cluster is congested and begins to take longer to respond than the configured request_timeout, the fluentd elasticsearch plugin will re-send the same bulk request. Since Elasticsearch can't tell its actually the same request, all documents in the request are indexed again resulting in duplicate data. In certain scenarios, this can result in essentially and infinite loop generating multiple copies of the same data.
 
-Using an _id generated by the fluentd elasticsearch plugin will communicate to Elasticsearch the uniqueness of the requests so that duplicates will be rejected or simply replace the existing records.
+The bundled elasticsearch_genid filter can generate a unique _hash key for each record, this key may be passed to the id_key parameter in the elasticsearch plugin to communicate to Elasticsearch the uniqueness of the requests so that duplicates will be rejected or simply replace the existing records.
 Here is a sample config:
 
 ```
-<hash>
-  hash_id_key _id    # storing generated hash id key
-  hash_id_key _hash    # storing generated hash id key
-</hash>
-id_key _hash # specify same key name which is specified in hash_id_key
+<filter>
+  @type elasticsearch_genid
+  hash_id_key _hash    # storing generated hash id key (default is _hash)
+</filter>
+<match **>
+  @type elasticsearch
+  id_key _hash # specify same key name which is specified in hash_id_key
+  remove_keys _hash # Elasticsearch doesn't like keys that start with _
+  # other settings are ommitted.
+</match>
 ```
 
 ### Not seeing a config you need?

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.1.1'
+  s.version       = '2.2.0.rc.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -0,0 +1,25 @@
+require 'securerandom'
+require 'base64'
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+  class ElasticsearchGenidFilter < Filter
+    Fluent::Plugin.register_filter('elasticsearch_genid', self)
+
+    config_param :hash_id_key, :string, :default => '_hash'
+
+    def initialize
+      super
+    end
+
+    def configure(conf)
+      super
+    end
+
+    def filter(tag, time, record)
+      record[@hash_id_key] = Base64.strict_encode64(SecureRandom.uuid)
+      record
+    end
+
+  end
+end

data/lib/fluent/plugin/generate_hash_id_support.rb

@@ -7,7 +7,8 @@ module Fluent
       def self.included(klass)
         klass.instance_eval {
           config_section :hash, param_name: :hash_config, required: false, multi: false do
-            config_param :hash_id_key, :string, default: '_hash'
+            config_param :hash_id_key, :string, default: '_hash',
+                         obsoleted: "Use bundled filer-elasticsearch-genid instead."
           end
         }
       end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -0,0 +1,44 @@
+require 'helper'
+require 'date'
+require 'fluent/test/helpers'
+require 'json'
+require 'fluent/test/driver/filter'
+require 'flexmock/test_unit'
+require 'fluent/plugin/filter_elasticsearch_genid'
+
+class ElasticsearchGenidFilterTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf='')
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
+  end
+
+  def sample_record
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal '_hash', d.instance.hash_id_key
+  end
+
+  data("default" => {"hash_id_key" => "_hash"},
+       "custom_key" => {"hash_id_key" => "_edited"},
+      )
+  def test_filter(data)
+    d = create_driver("hash_id_key #{data["hash_id_key"]}")
+    flexmock(SecureRandom).should_receive(:uuid)
+      .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
+    time = event_time("2017-10-15 15:00:23.34567890 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, sample_record)
+    end
+    assert_equal(Base64.strict_encode64(SecureRandom.uuid),
+                 d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+  end
+end

data/test/plugin/test_out_elasticsearch.rb

@@ -228,7 +228,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   test 'with invaild generate id config' do
-    assert_raise(Fluent::ConfigError) {
+    assert_raise_message(/Use bundled filer-elasticsearch-genid instead./) do
       driver.configure(Fluent::Config::Element.new(
                          'ROOT', '', {
                            '@type' => 'elasticsearch',
@@ -239,7 +239,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                                                        }, [])
                          ]
                        ))
-    }
+    end
   end
 
   def test_template_already_present
@@ -585,26 +585,19 @@ class ElasticsearchOutput < Test::Unit::TestCase
          "custom hash_id_key" => {"hash_id_key" => '_hash_id'},
         )
     def test_writes_with_genrate_hash(data)
-      driver.configure(Fluent::Config::Element.new(
-                         'ROOT', '', {
-                           '@type' => 'elasticsearch',
-                           'id_key' => data["hash_id_key"],
-                         }, [
-                           Fluent::Config::Element.new('hash', '', {
-                                                         'keys' => ['request_id'],
-                                                         'hash_id_key' => data["hash_id_key"],
-                                                       }, [])
-                         ]
-                       ))
-      stub_elastic_ping
-      stub_elastic
-      flexmock(SecureRandom).should_receive(:uuid)
-        .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
-      time = event_time("2017-10-15 15:00:23.34567890 UTC")
-      driver.run(default_tag: 'test') do
-        driver.feed(time, sample_record.merge('request_id' => 'elastic'))
+      assert_raise_message(/Use bundled filer-elasticsearch-genid instead./) do
+        driver.configure(Fluent::Config::Element.new(
+                           'ROOT', '', {
+                             '@type' => 'elasticsearch',
+                             'id_key' => data["hash_id_key"],
+                           }, [
+                             Fluent::Config::Element.new('hash', '', {
+                                                           'keys' => ['request_id'],
+                                                           'hash_id_key' => data["hash_id_key"],
+                                                         }, [])
+                           ]
+                         ))
       end
-      assert_equal(Base64.strict_encode64(SecureRandom.uuid), index_cmds[1]["#{data["hash_id_key"]}"])
     end
   end
 

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -323,27 +323,19 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
          "custom hash_id_key" => {"hash_id_key" => '_hash_id'},
         )
     def test_writes_with_genrate_hash(data)
-      driver.configure(Fluent::Config::Element.new(
-                         'ROOT', '', {
-                           '@type' => 'elasticsearch',
-                           'id_key' => data["hash_id_key"],
-                         }, [
-                           Fluent::Config::Element.new('hash', '', {
-                                                         'keys' => ['request_id'],
-                                                         'hash_id_key' => data["hash_id_key"],
-                                                       }, [])
-                         ]
-                       ))
-      stub_elastic_ping
-      stub_elastic
-      stub_elastic
-      flexmock(SecureRandom).should_receive(:uuid)
-        .and_return("82120f33-897a-4d9d-b3d5-14afd18fb412")
-      time = event_time("2017-10-15 15:00:23.34567890 UTC")
-      driver.run(default_tag: 'test') do
-        driver.feed(time, sample_record.merge('request_id' => 'elastic'))
+      assert_raise_message(/Use bundled filer-elasticsearch-genid instead./) do
+        driver.configure(Fluent::Config::Element.new(
+                           'ROOT', '', {
+                             '@type' => 'elasticsearch',
+                             'id_key' => data["hash_id_key"],
+                           }, [
+                             Fluent::Config::Element.new('hash', '', {
+                                                           'keys' => ['request_id'],
+                                                           'hash_id_key' => data["hash_id_key"],
+                                                         }, [])
+                           ]
+                         ))
       end
-      assert_equal(Base64.strict_encode64(SecureRandom.uuid), index_cmds[1]["#{data["hash_id_key"]}"])
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0.rc.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-21 00:00:00.000000000 Z
+date: 2017-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -146,10 +146,12 @@ files:
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
+- lib/fluent/plugin/filter_elasticsearch_genid.rb
 - lib/fluent/plugin/generate_hash_id_support.rb
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
+- test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json
@@ -168,9 +170,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13
@@ -179,6 +181,7 @@ specification_version: 4
 summary: ElasticSearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json