fluent-plugin-elasticsearch 2.11.2 → 2.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a4e029357f6bb2c914d2754eb9752a88d774e9eb326cb10f52a6d56d3e3ce09
-  data.tar.gz: b756fbd56d5c830df97f32b97d3f9f9f8593253c59a9861a7d3f8c2ae9d47220
+  metadata.gz: ff924cb0d3a10af63436dc4ccf10256530637857bc3cc64dcef2cb59473a15a9
+  data.tar.gz: 7f6615d23e6ead3b22abe866036f79f93f1acfb77a87a1251b1ccd6511df01e1
 SHA512:
-  metadata.gz: 6fa642ed492678b604eb7e8ff5a51e3f03cbe27929d64608432ee538ab98c3f6ca42f5dca63c6434d3290c977f3740445dbb9c3742d7d85faf43603432a491a1
-  data.tar.gz: 843ad9a1fcc3174871e1762a1843839ef9291bb2e862fee331f0b5f8b79f770273295d3c5fef0b0897708b325345110e19afe8a346dd99126c8ba7f0d2cfd98e
+  metadata.gz: 1004359a2090e989ebc1906cb198388178d0556859d07cd051fce90f248d2d2e02c247fdb5f29714aa77e29634e5649d4bac11861d01beb3da92574bd75fe9be
+  data.tar.gz: 0067bfcfc7efc89a9d48c8c3d83c1e0ff2a430c413d61a7ff4dace664832d4f1a1443b90c0066113313e315b736183278bbb55f7955424dcc984de047e3ed331

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 2.11.3
+- Implement the `include_index_in_url` option for out_elasticsearch (#451)
+- Add an option `include_index_in_url` to allow URL-based conrtrols (#450)
+
 ### 2.11.2
 - Strictness scheme (#445)
 

data/README.md

@@ -55,6 +55,7 @@ Current maintainers: @cosmo0920
   + [reconnect_on_error](#reconnect_on_error)
   + [with_transporter_log](#with_transporter_log)
   + [content_type](#content_type)
+  + [include_index_in_url](#include_index_in_url)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
   + [Buffer options](#buffer-options)
@@ -614,6 +615,15 @@ If you will not use template, it recommends to set `content_type application/x-n
 content_type application/x-ndjson
 ```
 
+### include_index_in_url
+
+With this option set to true, Fluentd manifests the index name in the request URL (rather than in the request body).
+You can use this option to enforce an URL-based access control.
+
+```
+include_index_in_url true
+```
+
 ### Client/host certificate options
 
 Need to verify Elasticsearch's certificate?  You can use the following parameter to specify a CA instead of using an environment variable.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.11.2'
+  s.version       = '2.11.3'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -32,7 +32,8 @@ class Fluent::Plugin::ElasticsearchErrorHandler
       begin
         # we need a deep copy for process_message to alter
         processrecord = Marshal.load(Marshal.dump(rawrecord))
-        next unless @plugin.process_message(tag, meta, header, time, processrecord, bulk_message, extracted_values)
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
+        next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
       rescue => e
         stats[:bad_chunk_record] += 1
         next

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -34,6 +34,8 @@ module Fluent::Plugin
       end
     end
 
+    RequestInfo = Struct.new(:host, :index)
+
     helpers :event_emitter, :compat_parameters, :record_accessor
 
     Fluent::Plugin.register_output('elasticsearch', self)
@@ -104,6 +106,7 @@ EOC
 elasticsearch gem v6.0.2 starts to use correct Content-Type. Please upgrade elasticserach gem and stop to use this option.
 see: https://github.com/elastic/elasticsearch-ruby/pull/514
 EOC
+    config_param :include_index_in_url, :bool, :default => false
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -420,8 +423,8 @@ EOC
     end
 
     def write(chunk)
-      bulk_message_count = 0
-      bulk_message = ''
+      bulk_message_count = Hash.new { |h,k| h[k] = 0 }
+      bulk_message = Hash.new { |h,k| h[k] = '' }
       header = {}
       meta = {}
 
@@ -432,8 +435,15 @@ EOC
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
         begin
-          if process_message(tag, meta, header, time, record, bulk_message, extracted_values)
-            bulk_message_count += 1
+          meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
+          info = if @include_index_in_url
+                   RequestInfo.new(nil, meta.delete("_index".freeze))
+                 else
+                   RequestInfo.new(nil, nil)
+                 end
+
+          if append_record_to_messages(@write_operation, meta, header, record, bulk_message[info])
+            bulk_message_count[info] += 1;
           else
             if @emit_error_for_missing_id
               raise MissingIdFieldError, "Missing '_id' field. Write operation is #{@write_operation}"
@@ -446,11 +456,14 @@ EOC
         end
       end
 
-      send_bulk(bulk_message, tag, chunk, bulk_message_count, extracted_values) unless bulk_message.empty?
-      bulk_message.clear
+
+      bulk_message.each do |info, msgs|
+        send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info.index) unless msgs.empty?
+        msgs.clear
+      end
     end
 
-    def process_message(tag, meta, header, time, record, bulk_message, extracted_values)
+    def process_message(tag, meta, header, time, record, extracted_values)
       logstash_prefix, index_name, type_name = extracted_values
 
       if @flatten_hashes
@@ -530,7 +543,7 @@ EOC
         @remove_keys.each { |key| record.delete(key) }
       end
 
-      append_record_to_messages(@write_operation, meta, header, record, bulk_message)
+      return [meta, header, record]
     end
 
     # returns [parent, child_key] of child described by path array in record's tree
@@ -542,12 +555,12 @@ EOC
 
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
-    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values)
+    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, index)
       retries = 0
       begin
 
         log.on_trace { log.trace "bulk request: #{data}" }
-        response = client.bulk body: data
+        response = client.bulk body: data, index: index
         log.on_trace { log.trace "bulk response: #{response}" }
 
         if response['errors']

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -13,6 +13,8 @@ module Fluent::Plugin
     DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
     DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
 
+    RequestInfo = Struct.new(:host, :index)
+
     attr_reader :dynamic_config
 
     def configure(conf)
@@ -186,7 +188,19 @@ module Fluent::Plugin
           record.merge!(dynamic_conf['tag_key'] => tag)
         end
 
-        meta = {"_index" => target_index, "_type" => dynamic_conf['type_name']}
+        if dynamic_conf['hosts']
+          host = dynamic_conf['hosts']
+        else
+          host = "#{dynamic_conf['host']}:#{dynamic_conf['port']}"
+        end
+
+        if @include_index_in_url
+          key = RequestInfo.new(host, target_index)
+          meta = {"_type" => dynamic_conf['type_name']}
+        else
+          key = RequestInfo.new(host, nil)
+          meta = {"_index" => target_index, "_type" => dynamic_conf['type_name']}
+        end
 
         @meta_config_map.each_pair do |config_name, meta_key|
           if dynamic_conf[config_name] && accessor = record_accessor_create(dynamic_conf[config_name])
@@ -196,30 +210,24 @@ module Fluent::Plugin
           end
         end
 
-        if dynamic_conf['hosts']
-          host = dynamic_conf['hosts']
-        else
-          host = "#{dynamic_conf['host']}:#{dynamic_conf['port']}"
-        end
-
         if @remove_keys
           @remove_keys.each { |key| record.delete(key) }
         end
 
         write_op = dynamic_conf["write_operation"]
-        append_record_to_messages(write_op, meta, headers[write_op], record, bulk_message[host])
+        append_record_to_messages(write_op, meta, headers[write_op], record, bulk_message[key])
       end
 
-      bulk_message.each do |hKey, msgs|
-        send_bulk(msgs, hKey) unless msgs.empty?
+      bulk_message.each do |info, msgs|
+        send_bulk(msgs, info.host, info.index) unless msgs.empty?
         msgs.clear
       end
     end
 
-    def send_bulk(data, host)
+    def send_bulk(data, host, index)
       retries = 0
       begin
-        response = client(host).bulk body: data
+        response = client(host).bulk body: data, index: index
         if response['errors']
           log.error "Could not push log to Elasticsearch: #{response}"
         end

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -22,7 +22,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
        @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
-    def process_message(tag, meta, header, time, record, bulk_message, extracted_values)
+    def process_message(tag, meta, header, time, record, extracted_values)
+      return [meta, header, record]
+    end
+
+    def append_record_to_messages(op, meta, header, record, msgs)
       if record.has_key?('raise') && record['raise']
         raise Exception('process_message')
       end

data/test/plugin/test_out_elasticsearch.rb

@@ -2174,4 +2174,18 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert(index_cmds[0].has_key?("create"))
   end
 
+  def test_include_index_in_url
+    stub_elastic_ping
+    stub_elastic('http://localhost:9200/logstash-2018.01.01/_bulk')
+
+    driver.configure("index_name logstash-2018.01.01
+                      include_index_in_url true")
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+
+    assert_equal(index_cmds.length, 2)
+    assert_equal(index_cmds.first['index']['_index'], nil)
+  end
+
 end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -990,4 +990,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     end
     assert(index_cmds[0].has_key?("create"))
   end
+
+  def test_include_index_in_url
+    stub_elastic_ping
+    stub_elastic('http://localhost:9200/logstash-2018.01.01/_bulk')
+
+    driver.configure("index_name logstash-2018.01.01
+                      include_index_in_url true")
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+
+    assert_equal(index_cmds.length, 2)
+    assert_equal(index_cmds.first['index']['_index'], nil)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.11.2
+  version: 2.11.3
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-20 00:00:00.000000000 Z
+date: 2018-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd