fluent-plugin-elasticsearch 2.10.5 → 2.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f9dd556dc8e7f51f181c44a3bcff4496bca22ee330c7a374654da3cc2772773
-  data.tar.gz: 76cc551a4dd10d6f4d5f215d4a736db3b1f9cb9ab7a7999166a9a4345a55c209
+  metadata.gz: b98a52f6ea81dcfb963cb072468dfdbbfd0ad23202c9974d1e19032ad3059b56
+  data.tar.gz: 79ab8568a0d34dd71adb9c8b3e1016bdad3cabda102603a3da49cd19f9f15e1e
 SHA512:
-  metadata.gz: 3221ab803aa9c873e9f9c8631a87899d8376adc8fa1a90627d3856eb929b0a304fa8850b0dd58d20e23b4a7020512ea4e16b9311f017199ef3e493b4fee9d4ab
-  data.tar.gz: 7e0e90b03e16d63b74e1936f33f1d6586dcab00b78c68b47342ea6044e96609f2129e38629821d34a06241d7caafedf6e47825601ff39c2410140e8e1696ee4d
+  metadata.gz: 843a03b70c2e6da558da1674b877b4771bbfe55f8ce444cd2c53a279b47772413c680059a09b57e420638f4bf6b91549f680177ef4881f32d4258c02780de105
+  data.tar.gz: 321d22427a0f2c0770dd48f13ebcc710390a44b908329853cac4f4b7c69e6eda0d31591af507c58b99e71bacbd70f9a9fc80a793a4651336f4c21cf43172ede7

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 2.11.0
+- Uplift Merge pull request #437 from jcantrill/fix_bulk_count (#438)
+
 ### 2.10.5
 - Uplift Merge pull request #435 from jcantrill/add_trace_logging (#436)
 

data/README.md

@@ -17,6 +17,7 @@ Current maintainers: @cosmo0920
 * [Usage](#usage)
   + [Index templates](#index-templates)
 * [Configuration](#configuration)
+  + [emit_error_for_missing_id](#emit_error_for_missing_id)
   + [hosts](#hosts)
   + [user, password, path, scheme, ssl_verify](#user-password-path-scheme-ssl_verify)
   + [logstash_format](#logstash_format)
@@ -104,6 +105,15 @@ This plugin creates Elasticsearch indices by merely writing to them. Consider us
 
 ## Configuration
 
+### emit_error_for_missing_id
+
+```
+emit_error_for_missing_id true
+```
+When  `write_operation` is configured to anything other then `index`, setting this value to `true` will
+cause the plugin to `emit_error_event` of any records which do not include an `_id` field.  The default (`false`)
+behavior is to silently drop the records.
+
 ### hosts
 
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.10.5'
+  s.version       = '2.11.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -32,7 +32,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
       begin
         # we need a deep copy for process_message to alter
         processrecord = Marshal.load(Marshal.dump(rawrecord))
-        @plugin.process_message(tag, meta, header, time, processrecord, bulk_message, extracted_values)
+        next unless @plugin.process_message(tag, meta, header, time, processrecord, bulk_message, extracted_values)
       rescue => e
         stats[:bad_chunk_record] += 1
         next

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -20,6 +20,10 @@ module Fluent::Plugin
   class ElasticsearchOutput < Output
     class ConnectionFailure < Fluent::UnrecoverableError; end
 
+    # MissingIdFieldError is raised for records that do not
+    # include the field for the unique record identifier
+    class MissingIdFieldError < StandardError; end
+
     # RetryStreamError privides a stream to be
     # put back in the pipeline for cases where a bulk request
     # failed (e.g some records succeed while others failed)
@@ -94,6 +98,7 @@ EOC
     config_param :reconnect_on_error, :bool, :default => false
     config_param :pipeline, :string, :default => nil
     config_param :with_transporter_log, :bool, :default => false
+    config_param :emit_error_for_missing_id, :bool, :default => false
     config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson"], :default => :"application/json",
                  :deprecated => <<EOC
 elasticsearch gem v6.0.2 starts to use correct Content-Type. Please upgrade elasticserach gem and stop to use this option.
@@ -324,6 +329,13 @@ EOC
       end.join(', ')
     end
 
+    # append_record_to_messages adds a record to the bulk message
+    # payload to be submitted to Elasticsearch.  Records that do
+    # not include '_id' field are skipped when 'write_operation'
+    # is configured for 'create' or 'update'
+    #
+    # returns 'true' if record was appended to the bulk message
+    #         and 'false' otherwise
     def append_record_to_messages(op, meta, header, record, msgs)
       case op
       when UPDATE_OP, UPSERT_OP
@@ -331,18 +343,22 @@ EOC
           header[UPDATE_OP] = meta
           msgs << @dump_proc.call(header) << BODY_DELIMITER
           msgs << @dump_proc.call(update_body(record, op)) << BODY_DELIMITER
+          return true
         end
       when CREATE_OP
         if meta.has_key?(ID_FIELD)
           header[CREATE_OP] = meta
           msgs << @dump_proc.call(header) << BODY_DELIMITER
           msgs << @dump_proc.call(record) << BODY_DELIMITER
+          return true
         end
       when INDEX_OP
         header[INDEX_OP] = meta
         msgs << @dump_proc.call(header) << BODY_DELIMITER
         msgs << @dump_proc.call(record) << BODY_DELIMITER
+        return true
       end
+      return false
     end
 
     def update_body(record, op)
@@ -406,8 +422,15 @@ EOC
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
         begin
-          process_message(tag, meta, header, time, record, bulk_message, extracted_values)
-          bulk_message_count += 1
+          if process_message(tag, meta, header, time, record, bulk_message, extracted_values)
+            bulk_message_count += 1
+          else
+            if @emit_error_for_missing_id
+              raise MissingIdFieldError, "Missing '_id' field. Write operation is #{@write_operation}"
+            else
+              log.on_debug { log.debug("Dropping record because its missing an '_id' field and write_operation is #{@write_operation}: #{record}") }
+            end
+          end
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -26,6 +26,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       if record.has_key?('raise') && record['raise']
         raise Exception('process_message')
       end
+      return true
     end
   end
 
@@ -54,7 +55,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
   end
 
   def test_dlq_400_responses
-    records = [{time: 123, record: {"foo" => "bar"}}]
+    records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
     response = parse_response(%({
       "took" : 0,
       "errors" : true,

data/test/plugin/test_out_elasticsearch.rb

@@ -41,8 +41,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
 
-  def sample_record
-    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
+  def sample_record(content={})
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
   end
 
   def nested_sample_record
@@ -178,9 +178,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
   end
 
-  def assert_logs_include(logs, msg)
+  def assert_logs_include(logs, msg, exp_matches=1)
     matches = logs.grep /#{msg}/
-    assert_equal(1, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
+    assert_equal(exp_matches, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
   end
 
   def test_configure
@@ -1799,6 +1799,106 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal [['retry', 1, sample_record]], driver.events
   end
 
+  def test_create_should_write_records_with_ids_and_skip_those_without
+    driver.configure("write_operation create\nid_key my_id\n@log_level debug")
+    stub_elastic_ping
+    stub_request(:post, 'http://localhost:9200/_bulk')
+        .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => %({
+          "took" : 1,
+          "errors" : true,
+          "items" : [
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc"
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "xyz",
+                "status" : 500,
+                "error" : {
+                  "type" : "some unrecognized type",
+                  "reason":"some error to cause version mismatch"
+                }
+              }
+            }
+           ]
+        })
+     }
+    end)
+    sample_record1 = sample_record('my_id' => 'abc')
+    sample_record4 = sample_record('my_id' => 'xyz')
+
+    driver.run(default_tag: 'test') do
+      driver.feed(1, sample_record1)
+      driver.feed(2, sample_record)
+      driver.feed(3, sample_record)
+      driver.feed(4, sample_record4)
+    end
+
+    logs = driver.logs
+    # one record succeeded while the other should be 'retried'
+    assert_equal [['test', 4, sample_record4]], driver.events
+    assert_logs_include(logs, /(Dropping record)/, 2)
+  end
+
+  def test_create_should_write_records_with_ids_and_emit_those_without
+    driver.configure("write_operation create\nid_key my_id\nemit_error_for_missing_id true\n@log_level debug")
+    stub_elastic_ping
+    stub_request(:post, 'http://localhost:9200/_bulk')
+        .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => %({
+          "took" : 1,
+          "errors" : true,
+          "items" : [
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc"
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "xyz",
+                "status" : 500,
+                "error" : {
+                  "type" : "some unrecognized type",
+                  "reason":"some error to cause version mismatch"
+                }
+              }
+            }
+           ]
+        })
+     }
+    end)
+    sample_record1 = sample_record('my_id' => 'abc')
+    sample_record4 = sample_record('my_id' => 'xyz')
+
+    driver.run(default_tag: 'test') do
+      driver.feed(1, sample_record1)
+      driver.feed(2, sample_record)
+      driver.feed(3, sample_record)
+      driver.feed(4, sample_record4)
+    end
+
+    error_log = driver.error_events.map {|e| e.last.message }
+    # one record succeeded while the other should be 'retried'
+    assert_equal [['test', 4, sample_record4]], driver.events
+    assert_logs_include(error_log, /(Missing '_id' field)/, 2)
+  end
+
   def test_bulk_error
     stub_elastic_ping
     stub_request(:post, 'http://localhost:9200/_bulk')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.10.5
+  version: 2.11.0
 platform: ruby
 authors:
 - diogo