fluent-plugin-elasticsearch 3.2.4 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87e8dce0d946717ddad6f80e9c43682ea3e66d470ce7593f3ddc894c2f2efa59
-  data.tar.gz: 2721ab242a253c8d14e50b37a6484d5d51b240b2561ff872e43a1c5a6b943eec
+  metadata.gz: '029a448c4a3a928294dd5bf42e6cfd307b0ec035f6450255f239357413321247'
+  data.tar.gz: c88e603bce04eb4d1e34e347bd1b97d8a799c16c4149900ecd9e2b7cbc2dbcce
 SHA512:
-  metadata.gz: bc68a120c99d8951c74bbe91e283a6229539c5b8a5d5afb41ccc4aea76ed528459b6debee8843e62547628bba57d243e87ac7a6aea0389ea617610c5ed1ad374
-  data.tar.gz: c2a7f70b7cc0ee50f0344775662969fddff4812cc01652e894739eb72fe5ee0fcf1483a53bbabc7067f509df3084988dbbe9cad5166891394e722b90ff5af6d1
+  metadata.gz: 964c3453011c608aaaaf78c7d7af899d5a981036f70fb8d3c1dd592b450010a447a29638486732c19af4802aeb9239dc1402652f094cda7c3ac834fb495dc860
+  data.tar.gz: 0615267ec3802f4e7f8693c50628cea67c648da38970dfbc54a01b54099f3446a00f2f924efd7354a954bfc521d5894fcd65fc3ad8aac2f3f4050ce23dbd6dd6

data/History.md

@@ -1,6 +1,10 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+
+### 3.3.0
+- Support builtin placeholders for host and hosts parameter (#554)
+
 ### 3.2.4
 - Pass chunk for built in placeholders (#553)
 

data/README.md

@@ -15,6 +15,7 @@ Current maintainers: @cosmo0920
 * [Usage](#usage)
   + [Index templates](#index-templates)
 * [Configuration](#configuration)
+  + [host](#host)
   + [emit_error_for_missing_id](#emit_error_for_missing_id)
   + [hosts](#hosts)
   + [user, password, path, scheme, ssl_verify](#user-password-path-scheme-ssl_verify)
@@ -128,6 +129,16 @@ This plugin creates Elasticsearch indices by merely writing to them. Consider us
 
 ## Configuration
 
+### host
+
+```
+host user-custom-host.domain # default localhost
+```
+
+You can specify Elasticsearch host by this parameter.
+
+**Note:** Since v3.3.0, `host` parameter supports builtin placeholders. If you want to send events dinamically into different hosts at runtime with `elasticsearch_dynamic` output plugin, please consider to switch to use plain `elasticsearch` output plugin. In more detail for builtin placeholders, please refer to [Placeholders](#placeholders) section.
+
 ### emit_error_for_missing_id
 
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.2.4'
+  s.version       = '3.3.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -348,7 +348,10 @@ EOC
       return Time.at(event_time).to_datetime
     end
 
-    def client
+    def client(host = nil)
+      # check here to see if we already have a client connection for the given host
+      connection_options = get_connection_options(host)
+
       @_es ||= begin
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
         local_reload_connections = @reload_connections
@@ -356,7 +359,7 @@ EOC
           local_reload_connections = @reload_after
         end
         headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers)
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
+        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: local_reload_connections,
                                                                               reload_on_failure: @reload_on_failure,
@@ -390,11 +393,11 @@ EOC
       end
     end
 
-    def get_connection_options
+    def get_connection_options(con_host=nil)
       raise "`password` must be present if `user` is present" if @user && !@password
 
-      hosts = if @hosts
-        @hosts.split(',').map do |host_str|
+      hosts = if con_host || @hosts
+        (con_host || @hosts).split(',').map do |host_str|
           # Support legacy hosts format host:port,host:port,host:port...
           if host_str.match(%r{^[^:]+(\:\d+)?$})
             {
@@ -519,22 +522,27 @@ EOC
 
       tag = chunk.metadata.tag
       extracted_values = expand_placeholders(chunk)
+      host = if @hosts
+               extract_placeholders(@hosts, chunk)
+             else
+               extract_placeholders(@host, chunk)
+             end
 
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
         begin
           meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
           info = if @include_index_in_url
-                   RequestInfo.new(nil, meta.delete("_index".freeze))
+                   RequestInfo.new(host, meta.delete("_index".freeze))
                  else
-                   RequestInfo.new(nil, nil)
+                   RequestInfo.new(host, nil)
                  end
 
           if append_record_to_messages(@write_operation, meta, header, record, bulk_message[info])
             bulk_message_count[info] += 1;
             if bulk_message[info].size > TARGET_BULK_BYTES
               bulk_message.each do |info, msgs|
-                send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info.index) unless msgs.empty?
+                send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
                 msgs.clear
                 # Clear bulk_message_count for this info.
                 bulk_message_count[info] = 0;
@@ -555,7 +563,7 @@ EOC
 
 
       bulk_message.each do |info, msgs|
-        send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info.index) unless msgs.empty?
+        send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
         msgs.clear
       end
     end
@@ -648,11 +656,11 @@ EOC
 
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
-    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, index)
+    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
       begin
 
         log.on_trace { log.trace "bulk request: #{data}" }
-        response = client.bulk body: data, index: index
+        response = client(info.host).bulk body: data, index: info.index
         log.on_trace { log.trace "bulk response: #{response}" }
 
         if response['errors']

data/test/plugin/test_out_elasticsearch.rb

@@ -1462,6 +1462,88 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  class HostnamePlaceholders < self
+    def test_writes_to_extracted_host
+      driver.configure("host ${tag}\n")
+      time = Time.parse Date.today.iso8601
+      elastic_request = stub_elastic("http://extracted-host:9200/_bulk")
+      driver.run(default_tag: 'extracted-host') do
+        driver.feed(time.to_i, sample_record)
+      end
+      assert_requested(elastic_request)
+    end
+
+    def test_writes_to_multi_hosts_with_placeholders
+      hosts = [['${tag}', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
+      hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
+
+      driver.configure("hosts #{hosts_string}")
+
+      hosts.each do |host_info|
+        host, port = host_info
+        host = "extracted-host" if host == '${tag}'
+        stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
+      end
+
+      driver.run(default_tag: 'extracted-host') do
+        1000.times do
+          driver.feed(sample_record.merge('age'=>rand(100)))
+        end
+      end
+
+      # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
+      # it's Fluentd test driver's constraint
+      # so @index_command_counts.size is always 1
+
+      assert(@index_command_counts.size > 0, "not working with hosts options")
+
+      total = 0
+      @index_command_counts.each do |url, count|
+        total += count
+      end
+      assert_equal(2000, total)
+    end
+
+    def test_writes_to_extracted_host_with_time_placeholder
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'host' => 'host-%Y%m%d',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,time', {
+                                                         'chunk_keys' => ['tag', 'time'],
+                                                         'timekey' => 3600,
+                                                       }, [])
+                         ]
+                       ))
+      stub_elastic
+      time = Time.parse Date.today.iso8601
+      elastic_request = stub_elastic("http://host-#{time.utc.strftime('%Y%m%d')}:9200/_bulk")
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record)
+      end
+      assert_requested(elastic_request)
+    end
+
+    def test_writes_to_extracted_host_with_custom_key_placeholder
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'host' => 'myhost-${pipeline_id}',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.iso8601
+      pipeline_id = "5"
+      elastic_request = stub_elastic("http://myhost-5:9200/_bulk")
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
+      end
+      assert_requested(elastic_request)
+    end
+  end
+
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
     driver.configure("logstash_format true
                       logstash_prefix MyPrefix")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.2.4
+  version: 3.3.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-26 00:00:00.000000000 Z
+date: 2019-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd