fluent-plugin-elasticsearch 3.3.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '029a448c4a3a928294dd5bf42e6cfd307b0ec035f6450255f239357413321247'
-  data.tar.gz: c88e603bce04eb4d1e34e347bd1b97d8a799c16c4149900ecd9e2b7cbc2dbcce
+  metadata.gz: 31cc9563debdbede8ae6e51e6e43089150c5ce736457433a9f4e8dc5652fb489
+  data.tar.gz: 021c1bde5bc10ffcd24536a7fcda38151dcc2508edcfb87b973dee8cf8f1fdd4
 SHA512:
-  metadata.gz: 964c3453011c608aaaaf78c7d7af899d5a981036f70fb8d3c1dd592b450010a447a29638486732c19af4802aeb9239dc1402652f094cda7c3ac834fb495dc860
-  data.tar.gz: 0615267ec3802f4e7f8693c50628cea67c648da38970dfbc54a01b54099f3446a00f2f924efd7354a954bfc521d5894fcd65fc3ad8aac2f3f4050ce23dbd6dd6
+  metadata.gz: faca5fb42a3a126f27b864a48588a1fd918b645b16287500b2793150ed83fe5d2378f1c0527917fe33309a9fcb7af0f7645190700043f30fd8a02d3fd04ddcf6
+  data.tar.gz: 363df0da34ce21414a993c4464ba60766c3e0b0dd171b61674d6bd8850d3ca8d4055563d231439b1004cfc5aaa0b54f311e9002a6689c938960f011ab70e051a

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 3.3.1
+- add new option to suppress doc wrapping (#557)
+- Include 2 (#555)
+
 ### 3.3.0
 - Support builtin placeholders for host and hosts parameter (#554)
 

data/README.md

@@ -81,6 +81,7 @@ Current maintainers: @cosmo0920
   + [Placeholders](#placeholders)
   + [Multi workers](#multi-workers)
   + [log_es_400_reason](#log_es_400_reason)
+  + [suppress_doc_wrap](#suppress_doc_wrap)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -1025,6 +1026,12 @@ By default, the error logger won't record the reason for a 400 error from the El
 
 Default value is `false`.
 
+## suppress_doc_wrap
+
+By default, record body is wrapped by 'doc'. This behavior can not handle update script requests. You can set this to suppress doc wrapping and allow record body to be untouched.
+
+Default value is `false`.
+
 ## Troubleshooting
 
 ### Cannot send events to Elasticsearch
@@ -1265,7 +1272,7 @@ The following configuration uses label:
 fluent-plugin-elasticsearch default behavior has a possibility to cause events traffic jam.
 When users use `flush_thread_count` = 1, ES plugin retries to send events if connection errors are disappeared.
 
-To prevent the following warning and sending events blocking, you must specify `flush_thread_count` > 2:
+To prevent the following warning and sending events blocking, you must specify `flush_thread_count` >= 2:
 
 ```log
 2018-12-24 14:32:06 +0900 [warn]: #0 To prevent events traffic jam, you should specify 2 or more 'flush_thread_count'.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.3.0'
+  s.version       = '3.3.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -132,6 +132,7 @@ EOC
     config_param :default_elasticsearch_version, :integer, :default => DEFAULT_ELASTICSEARCH_VERSION
     config_param :log_es_400_reason, :bool, :default => false
     config_param :custom_headers, :hash, :default => {}
+    config_param :suppress_doc_wrap, :bool, :default => false
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -468,6 +469,9 @@ EOC
 
     def update_body(record, op)
       update = remove_keys(record)
+      if @suppress_doc_wrap
+        return update
+      end
       body = {"doc".freeze => update}
       if op == UPSERT_OP
         if update == record

data/test/plugin/test_out_elasticsearch.rb

@@ -2510,4 +2510,59 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_logs_include_compare_size(1, "<=", log, /In Fluent::Plugin::ElasticsearchSimpleSniffer hosts/)
   end
 
+  def test_suppress_doc_wrap
+    driver.configure('write_operation update
+                      id_key id
+                      remove_keys id
+                      suppress_doc_wrap true')
+    stub_elastic
+    doc_body = {'field' => 'value'}
+    script_body = {'source' => 'ctx._source.counter += params.param1',
+                   'lang' => 'painless',
+                   'params' => {'param1' => 1}}
+    upsert_body = {'counter' => 1}
+    driver.run(default_tag: 'test') do
+      driver.feed('id' => 1, 'doc' => doc_body)
+      driver.feed('id' => 2, 'script' => script_body, 'upsert' => upsert_body)
+    end
+    assert(
+      index_cmds[1] == {'doc' => doc_body}
+    )
+    assert(
+      index_cmds[3] == {
+        'script' => script_body,
+        'upsert' => upsert_body
+      }
+    )
+  end
+
+  def test_suppress_doc_wrap_should_handle_record_as_is_at_upsert
+    driver.configure('write_operation upsert
+                      id_key id
+                      remove_keys id
+                      suppress_doc_wrap true')
+    stub_elastic
+    doc_body = {'field' => 'value'}
+    script_body = {'source' => 'ctx._source.counter += params.param1',
+                   'lang' => 'painless',
+                   'params' => {'param1' => 1}}
+    upsert_body = {'counter' => 1}
+    driver.run(default_tag: 'test') do
+      driver.feed('id' => 1, 'doc' => doc_body, 'doc_as_upsert' => true)
+      driver.feed('id' => 2, 'script' => script_body, 'upsert' => upsert_body)
+    end
+    assert(
+      index_cmds[1] == {
+        'doc' => doc_body,
+        'doc_as_upsert' => true
+      }
+    )
+    assert(
+      index_cmds[3] == {
+        'script' => script_body,
+        'upsert' => upsert_body
+      }
+    )
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.3.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-01 00:00:00.000000000 Z
+date: 2019-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd