fluent-plugin-elasticsearch 1.9.1 → 1.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f78bc5cd41e0fbe35c7732472cf1f12acfe5ef2
-  data.tar.gz: 2aec9b3232cde4c37389164354f55b2f5d07ba36
+  metadata.gz: e4f1361244893fa6e604c50f5b85d8627fa4422d
+  data.tar.gz: 6adfbf2916a9383e11865724a8400c3a0d6ea208
 SHA512:
-  metadata.gz: 9d0c83d28e364bf3c9372cad94a4f345a1f444c0eba3bbdb62bfb5ecb000ced86174b294c212d4518fcd56a781d8f2c193d6799416fe4a6a93ca5d8606d7ee0d
-  data.tar.gz: 89345bbcb03e101010b28a3d1e10928f8ce4c2a648a613c739a5ea3bf3e46445d09f2f66b820767e54150d22ccab659ee07f3816bd6d4ed27ab8d03691b33f25
+  metadata.gz: cc662aea89454d7136e056eacfcfcbcd12a704afa23ed45b4998642e5922eac524cea113c3afd0b0124ba15709e9b6844d9910a2e88e39b7fc07ec8202b5067a
+  data.tar.gz: 6cea6d23ae9c9fb779dab7492cf2dd444252de7cb586813c0cd3807a36d0b0a97ab07ba95e0da26e52a129e932a1b3ff3981397cbae67ac3def53046bc2610df

data/.travis.yml

@@ -5,5 +5,9 @@ rvm:
   - 2.2
   - 2.3.1
 
+gemfile:
+ - Gemfile
+ - Gemfile.v0.12
+
 script: bundle exec rake test
 sudo: false

data/Gemfile.v0.12

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
+gem 'fluentd', '~> 0.12.0'
+
+gemspec
+
+
+gem 'simplecov', require: false
+gem 'coveralls', require: false
+gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 1.9.2
+- Fix elasticsearch_dynamic for v0.14 (#224)
+
 ### 1.9.1
 - Cast `reload_*` configs in out_elasticsearch_dynamic to bool (#220)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.9.1'
+  s.version       = '1.9.2'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -7,28 +7,20 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
 
   config_param :delimiter, :string, :default => "."
 
-  # params overloaded as strings
-  config_param :port, :string, :default => "9200"
-  config_param :logstash_format, :string, :default => "false"
-  config_param :utc_index, :string, :default => "true"
-  config_param :time_key_exclude_timestamp, :bool, :default => false
-  config_param :reload_connections, :string, :default => "true"
-  config_param :reload_on_failure, :string, :default => "false"
-  config_param :resurrect_after, :string, :default => "60"
-  config_param :ssl_verify, :string, :default => "true"
-  config_param :reconnect_on_error, :bool, :default => false
+  DYNAMIC_PARAM_NAMES = %W[hosts host port logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
+  DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
+
+  attr_reader :dynamic_config
 
   def configure(conf)
     super
 
     # evaluate all configurations here
-    @dynamic_params ||= []
-    @dynamic_params += self.instance_variables.select { |var| is_valid_expand_param_type(var) }
-    @dynamic_config = Hash.new
-    @dynamic_params.each { |var|
+    @dynamic_config = {}
+    DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
       value = expand_param(self.instance_variable_get(var), nil, nil, nil)
-      var = var[1..-1]
-      @dynamic_config[var] = value
+      key = DYNAMIC_PARAM_NAMES[i]
+      @dynamic_config[key] = value.to_s
     }
     # end eval all configs
     @current_config = nil
@@ -47,17 +39,17 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
 
     @_es ||= begin
       @current_config = connection_options[:hosts].clone
-      excon_options = { client_key: @dynamic_config['client_key'], client_cert: @dynamic_config['client_cert'], client_key_pass: @dynamic_config['client_key_pass'] }
+      excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
       adapter_conf = lambda {|f| f.adapter :excon, excon_options }
       transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                           options: {
-                                                                            reload_connections: Fluent::Config.bool_value(@dynamic_config['reload_connections']),
-                                                                            reload_on_failure: Fluent::Config.bool_value(@dynamic_config['reload_on_failure']),
-                                                                            resurrect_after: @dynamic_config['resurrect_after'].to_i,
+                                                                            reload_connections: @reload_connections,
+                                                                            reload_on_failure: @reload_on_failure,
+                                                                            resurrect_after: @resurrect_after,
                                                                             retry_on_failure: 5,
                                                                             transport_options: {
-                                                                              request: { timeout: @dynamic_config['request_timeout'] },
-                                                                              ssl: { verify: @dynamic_config['ssl_verify'], ca_file: @dynamic_config['ca_file'] }
+                                                                              request: { timeout: @request_timeout },
+                                                                              ssl: { verify: @ssl_verify, ca_file: @ca_file }
                                                                             }
                                                                           }), &adapter_conf)
       es = Elasticsearch::Client.new transport: transport
@@ -74,16 +66,16 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
   end
 
   def get_connection_options(con_host)
-    raise "`password` must be present if `user` is present" if @dynamic_config['user'] && !@dynamic_config['password']
+    raise "`password` must be present if `user` is present" if @user && !@password
 
-    hosts = if con_host || @dynamic_config['hosts']
-      (con_host || @dynamic_config['hosts']).split(',').map do |host_str|
+    hosts = if con_host || @hosts
+      (con_host || @hosts).split(',').map do |host_str|
         # Support legacy hosts format host:port,host:port,host:port...
         if host_str.match(%r{^[^:]+(\:\d+)?$})
           {
             host:   host_str.split(':')[0],
-            port:   (host_str.split(':')[1] || @dynamic_config['port'] || @port).to_i,
-            scheme: @dynamic_config['scheme']
+            port:   (host_str.split(':')[1] || @port).to_i,
+            scheme: @scheme
           }
         else
           # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
@@ -95,10 +87,10 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         end
       end.compact
     else
-      [{host: @dynamic_config['host'], port: @dynamic_config['port'].to_i, scheme: @dynamic_config['scheme']}]
+      [{host: @host, port: @port.to_i, scheme: @scheme}]
     end.each do |host|
-      host.merge!(user: @dynamic_config['user'], password: @dynamic_config['password']) if !host[:user] && @dynamic_config['user']
-      host.merge!(path: @dynamic_config['path']) if !host[:path] && @dynamic_config['path']
+      host.merge!(user: @user, password: @password) if !host[:user] && @user
+      host.merge!(path: @path) if !host[:path] && @path
     end
 
     {
@@ -129,8 +121,8 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
       next unless record.is_a? Hash
 
       # evaluate all configurations here
-      @dynamic_params.each { |var|
-        k = var[1..-1]
+      DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
+        k = DYNAMIC_PARAM_NAMES[i]
         v = self.instance_variable_get(var)
         # check here to determine if we should evaluate
         if dynamic_conf[k] != v

data/test/plugin/test_out_elasticsearch.rb

@@ -13,7 +13,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def driver(tag='test', conf='')
-    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::ElasticsearchOutput, tag).configure(conf)
+    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::ElasticsearchOutput, tag) {
+      # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
+      if !defined?(Fluent::Plugin::Output)
+        def format(tag, time, record)
+          [time, record].to_msgpack
+        end
+      end
+    }.configure(conf)
   end
 
   def sample_record
@@ -1134,5 +1141,4 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
     assert(index_cmds[0].has_key?("create"))
   end
-
 end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -11,7 +11,14 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def driver(tag='test', conf='')
-    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::ElasticsearchOutputDynamic, tag).configure(conf)
+    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::ElasticsearchOutputDynamic, tag) {
+      # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
+      if !defined?(Fluent::Plugin::Output)
+        def format(tag, time, record)
+          [time, record].to_msgpack
+        end
+      end
+    }.configure(conf)
   end
 
   def sample_record
@@ -55,12 +62,12 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }
     instance = driver('test', config).instance
 
-    assert_equal 'logs.google.com', instance.host
-    assert_equal 777, instance.port
-    assert_equal 'https', instance.scheme
-    assert_equal '/es/', instance.path
+    conf = instance.dynamic_config
+    assert_equal 'logs.google.com', conf['host']
+    assert_equal "777", conf['port']
     assert_equal 'john', instance.user
     assert_equal 'doe', instance.password
+    assert_equal '/es/', instance.path
   end
 
   def test_defaults
@@ -73,14 +80,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }
     instance = driver('test', config).instance
 
-    assert_equal "9200", instance.port
-    assert_equal "false", instance.logstash_format
-    assert_equal "true", instance.utc_index
+    conf = instance.dynamic_config
+    assert_equal "9200", conf['port']
+    assert_equal "false", conf['logstash_format']
+    assert_equal "true", conf['utc_index']
     assert_equal false, instance.time_key_exclude_timestamp
-    assert_equal "true", instance.reload_connections
-    assert_equal "false", instance.reload_on_failure
-    assert_equal "60", instance.resurrect_after
-    assert_equal "true", instance.ssl_verify
   end
 
   def test_legacy_hosts_list

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.9.1
+  version: 1.9.2
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-14 00:00:00.000000000 Z
+date: 2016-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -122,6 +122,7 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
+- Gemfile.v0.12
 - History.md
 - ISSUE_TEMPLATE.md
 - LICENSE.txt