logstash-output-elasticsearch 2.1.2-java → 2.1.4-java

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -4,7 +4,7 @@ require "base64"
 require "elasticsearch"
 require "elasticsearch/transport/transport/http/manticore"
 
-module LogStash::Outputs::Elasticsearch
+module LogStash; module Outputs; class ElasticSearch;
   class HttpClient
     attr_reader :client, :options, :client_options, :sniffer_thread
     # This is here in case we use DEFAULT_OPTIONS in the future
@@ -30,6 +30,7 @@ module LogStash::Outputs::Elasticsearch
     end
 
     def bulk(actions)
+      return if actions.empty?
       bulk_body = actions.collect do |action, args, source|
         if action == 'update'
           if args[:_id]
@@ -150,4 +151,4 @@ module LogStash::Outputs::Elasticsearch
       @client.indices.put_template(:name => name, :body => template)
     end
   end
-end
+end end end

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -0,0 +1,92 @@
+module LogStash; module Outputs; class ElasticSearch;
+  module HttpClientBuilder
+    def self.build(logger, hosts, params)
+      client_settings = {}
+
+      common_options = {
+        :client_settings => client_settings,
+        :sniffing => params["sniffing"],
+        :sniffing_delay => params["sniffing_delay"]
+      }
+
+      common_options[:timeout] = params["timeout"] if params["timeout"]
+      client_settings[:path] = "/#{params["path"]}/".gsub(/\/+/, "/") # Normalize slashes
+      logger.debug? && logger.debug("Normalizing http path", :path => params["path"], :normalized => client_settings[:path])
+
+      client_settings.merge! setup_ssl(logger, params)
+      client_settings.merge! setup_proxy(logger, params)
+      common_options.merge! setup_basic_auth(logger, params)
+
+      # Update API setup
+      update_options = {
+        :upsert => params["upsert"],
+        :doc_as_upsert => params["doc_as_upsert"]
+      }
+      common_options.merge! update_options if params["action"] == 'update'
+
+      LogStash::Outputs::ElasticSearch::HttpClient.new(
+        common_options.merge(:hosts => hosts, :logger => logger)
+      )
+    end
+
+    def self.setup_proxy(logger, params)
+      proxy = params["proxy"]
+      return {} unless proxy
+
+      # Symbolize keys
+      proxy = if proxy.is_a?(Hash)
+                Hash[proxy.map {|k,v| [k.to_sym, v]}]
+              elsif proxy.is_a?(String)
+                proxy
+              else
+                raise LogStash::ConfigurationError, "Expected 'proxy' to be a string or hash, not '#{proxy}''!"
+              end
+
+      return {:proxy => proxy}
+    end
+
+    def self.setup_ssl(logger, params)
+      return {} unless params["ssl"]
+
+      cacert, truststore, truststore_password, keystore, keystore_password =
+        params.values_at('cacert', 'truststore', 'truststore_password', 'keystore', 'keystore_password')
+
+      if cacert && truststore
+        raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if truststore
+      end
+
+      ssl_options = {}
+
+      if cacert
+        ssl_options[:ca_file] = cacert
+      elsif truststore
+        ssl_options[:truststore_password] = truststore_password.value if truststore_password
+      end
+
+      ssl_options[:truststore] = truststore if truststore
+      if keystore
+        ssl_options[:keystore] = keystore
+        ssl_options[:keystore_password] = keystore_password.value if keystore_password
+      end
+      if !params["ssl_certificate_verification"]
+        logger.warn [
+                       "** WARNING ** Detected UNSAFE options in elasticsearch output configuration!",
+                       "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
+                       "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
+                     ].join("\n")
+        ssl_options[:verify] = false
+      end
+      { ssl: ssl_options }
+    end
+
+    def self.setup_basic_auth(logger, params)
+      user, password = params["user"], params["password"]
+      return {} unless user && password
+
+      {
+        :user => user,
+        :password => password.value
+      }
+    end
+  end
+end; end; end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -0,0 +1,35 @@
+module LogStash; module Outputs; class ElasticSearch
+  class TemplateManager
+    # To be mixed into the elasticsearch plugin base
+    def self.install_template(plugin)
+      return unless plugin.manage_template
+      plugin.logger.info("Using mapping template from", :path => plugin.template)
+      template = get_template(plugin.template)
+      plugin.logger.info("Attempting to install template", :manage_template => template)
+      install(plugin.client, plugin.template_name, template, plugin.template_overwrite)
+    rescue => e
+      plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name)
+    end
+
+    private
+
+    def self.get_template(path)
+      template_path = path || default_template_path
+      read_template_file(template_path)
+    end
+
+    def self.install(client, template_name, template, template_overwrite)
+      client.template_install(template_name, template, template_overwrite)
+    end
+
+    def self.default_template_path
+      ::File.expand_path('elasticsearch-template.json', ::File.dirname(__FILE__))
+    end
+
+    def self.read_template_file(template_path)
+      raise ArgumentError, "Template file '#{@template_path}' could not be found!" unless ::File.exists?(template_path)
+      template_data = ::IO.read(template_path)
+      LogStash::Json.load(template_data)
+    end
+  end
+end end end

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '2.1.2'
+  s.version         = '2.1.4'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'elasticsearch', ['>= 1.0.13', '~> 1.0']
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency 'cabin', ['~> 0.6']
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
 
   s.add_development_dependency 'ftw', '~> 0.0.42'
   s.add_development_dependency 'logstash-codec-plain'

data/spec/es_spec_helper.rb

@@ -13,8 +13,9 @@ CONTAINER_TAG = "1.6"
 DOCKER_INTEGRATION = ENV["DOCKER_INTEGRATION"]
 
 module ESHelper
-  def get_host
-    DOCKER_INTEGRATION ? Longshoreman.new.get_host_ip : "127.0.0.1"
+  def get_host_port
+    addr = DOCKER_INTEGRATION ? Longshoreman.new.get_host_ip : "127.0.0.1"
+    "#{addr}:#{get_port}"
   end
 
   def get_port
@@ -26,7 +27,7 @@ module ESHelper
   end
 
   def get_client
-    Elasticsearch::Client.new(:host => "#{get_host}:#{get_port}")
+    Elasticsearch::Client.new(:hosts => [get_host_port])
   end
 end
 
@@ -34,7 +35,6 @@ end
 RSpec.configure do |config|
   config.include ESHelper
 
-
   if DOCKER_INTEGRATION
     # this :all hook gets run before every describe block that is tagged with :integration => true.
     config.before(:all, :integration => true) do

data/spec/integration/outputs/create_spec.rb

@@ -9,8 +9,7 @@ describe "client create actions", :integration => true do
       "manage_template" => true,
       "index" => "logstash-create",
       "template_overwrite" => true,
-      "hosts" => get_host(),
-      "port" => get_port(),
+      "hosts" => get_host_port(),
       "action" => action
     }
     settings['document_id'] = id unless id.nil?
@@ -31,7 +30,7 @@ describe "client create actions", :integration => true do
       subject = get_es_output("create", "id123")
       subject.register
       subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.buffer_flush(:final => true)
+      subject.flush
       @es.indices.refresh
       # Wait or fail until everything's indexed.
       Stud::try(3.times) do
@@ -44,7 +43,7 @@ describe "client create actions", :integration => true do
       subject = get_es_output("create")
       subject.register
       subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.buffer_flush(:final => true)
+      subject.flush
       @es.indices.refresh
       # Wait or fail until everything's indexed.
       Stud::try(3.times) do

data/spec/integration/outputs/index_spec.rb

@@ -16,7 +16,7 @@ shared_examples "an indexer" do
     end
 
     it "ships events" do
-      index_url = "http://#{get_host}:#{get_port}/#{index}"
+      index_url = "http://#{get_host_port}/#{index}"
 
       ftw = FTW::Agent.new
       ftw.post!("#{index_url}/_refresh")
@@ -46,8 +46,7 @@ describe "an indexer with custom index_type", :integration => true do
   it_behaves_like "an indexer" do
     let(:config) {
       {
-        "hosts" => get_host,
-        "port" => get_port,
+        "hosts" => get_host_port,
         "index" => index,
         "flush_size" => flush_size
       }
@@ -60,8 +59,7 @@ describe "an indexer with no type value set (default to logs)", :integration =>
     let(:type) { "logs" }
     let(:config) {
       {
-        "hosts" => get_host,
-        "port" => get_port,
+        "hosts" => get_host_port,
         "index" => index,
         "flush_size" => flush_size
       }

data/spec/integration/outputs/retry_spec.rb

@@ -22,7 +22,7 @@ describe "failures in bulk class expected behavior", :integration => true do
       }
     end
 
-    allow_any_instance_of(LogStash::Outputs::Elasticsearch::HttpClient).to receive(:bulk).and_return(*expanded_responses)
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient).to receive(:bulk).and_return(*expanded_responses)
   end
 
   subject! do
@@ -30,8 +30,7 @@ describe "failures in bulk class expected behavior", :integration => true do
       "manage_template" => true,
       "index" => "logstash-2014.11.17",
       "template_overwrite" => true,
-      "hosts" => get_host(),
-      "port" => get_port(),
+      "hosts" => get_host_port(),
       "retry_max_items" => 10,
       "retry_max_interval" => 1,
       "max_retries" => max_retries
@@ -50,28 +49,34 @@ describe "failures in bulk class expected behavior", :integration => true do
     @es.indices.refresh
   end
 
+  after :each do
+    subject.close
+  end
+
   it "should return no errors if all bulk actions are successful" do
     mock_actions_with_response({"errors" => false})
     expect(subject).to receive(:submit).with([action1, action2]).once.and_call_original
     subject.register
     subject.receive(event1)
     subject.receive(event2)
-    subject.buffer_flush(:final => true)
+    subject.flush
     sleep(2)
   end
 
-  it "should raise exception and be retried by stud::buffer" do
+  it "retry exceptions within the submit body" do
     call_count = 0
-    expect(subject).to receive(:submit).with([action1]).exactly(3).times do
+    subject.register
+
+    expect(subject.client).to receive(:bulk).with(anything).exactly(3).times do
       if (call_count += 1) <= 2
         raise "error first two times"
       else
         {"errors" => false}
       end
     end
-    subject.register
+
     subject.receive(event1)
-    subject.close
+    subject.flush
   end
 
   it "should retry actions with response status of 503" do
@@ -87,17 +92,19 @@ describe "failures in bulk class expected behavior", :integration => true do
     subject.receive(event1)
     subject.receive(event1)
     subject.receive(event2)
-    subject.buffer_flush(:final => true)
+    subject.flush
     sleep(3)
   end
 
   it "should retry actions with response status of 429" do
+    subject.register
+
     mock_actions_with_response({"errors" => true, "statuses" => [429]},
                                {"errors" => false})
     expect(subject).to receive(:submit).with([action1]).twice.and_call_original
-    subject.register
+
     subject.receive(event1)
-    subject.buffer_flush(:final => true)
+    subject.flush
     sleep(3)
   end
 
@@ -108,17 +115,17 @@ describe "failures in bulk class expected behavior", :integration => true do
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]})
-    expect(subject).to receive(:submit).with([action1]).exactly(max_retries).times.and_call_original
+    expect(subject).to receive(:submit).with([action1]).exactly(max_retries+1).times.and_call_original
     subject.register
     subject.receive(event1)
-    subject.buffer_flush(:final => true)
-    sleep(3)
+    subject.flush
+    sleep(5)
   end
 
   it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunately)" do
     subject.register
     subject.receive(invalid_event)
-    expect(subject).not_to receive(:retry_push)
+    expect(subject).to receive(:submit).once.and_call_original
     subject.close
 
     @es.indices.refresh
@@ -132,7 +139,7 @@ describe "failures in bulk class expected behavior", :integration => true do
   it "successful requests should not be appended to retry queue" do
     subject.register
     subject.receive(event1)
-    expect(subject).not_to receive(:retry_push)
+    expect(subject).to receive(:submit).once.and_call_original
     subject.close
     @es.indices.refresh
     sleep(5)

data/spec/integration/outputs/routing_spec.rb

@@ -18,7 +18,7 @@ shared_examples "a routing indexer" do
 
 
     it "ships events" do
-      index_url = "http://#{get_host()}:#{get_port()}/#{index}"
+      index_url = "http://#{get_host_port()}/#{index}"
 
       ftw = FTW::Agent.new
       ftw.post!("#{index_url}/_refresh")
@@ -40,8 +40,7 @@ describe "(http protocol) index events with static routing", :integration => tru
     let(:routing) { "test" }
     let(:config) {
       {
-        "hosts" => get_host,
-        "port" => get_port,
+        "hosts" => get_host_port,
         "index" => index,
         "flush_size" => flush_size,
         "routing" => routing
@@ -55,8 +54,7 @@ describe "(http_protocol) index events with fieldref in routing value", :integra
     let(:routing) { "test" }
     let(:config) {
       {
-        "hosts" => get_host,
-        "port" => get_port,
+        "hosts" => get_host_port,
         "index" => index,
         "flush_size" => flush_size,
         "routing" => "%{message}"

data/spec/integration/outputs/secure_spec.rb

@@ -25,7 +25,7 @@ describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure =>
   it "sends events to ES" do
     expect {
       subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.buffer_flush(:final => true)
+      subject.flush
     }.to_not raise_error
   end
 end
@@ -49,7 +49,7 @@ describe "connect using HTTP Authentication", :elasticsearch_secure => true do
   it "sends events to ES" do
     expect {
       subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.buffer_flush(:final => true)
+      subject.flush
     }.to_not raise_error
   end
 end
@@ -79,7 +79,7 @@ describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure =>
   it "sends events to ES" do
     expect {
       subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.buffer_flush(:final => true)
+      subject.flush
     }.to_not raise_error
   end
 end
@@ -102,7 +102,7 @@ describe "connect using HTTP Authentication", :elasticsearch_secure => true do
   it "sends events to ES" do
     expect {
       subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.buffer_flush(:final => true)
+      subject.flush
     }.to_not raise_error
   end
 end

data/spec/integration/outputs/templates_spec.rb

@@ -6,8 +6,7 @@ describe "index template expected behavior", :integration => true do
     settings = {
       "manage_template" => true,
       "template_overwrite" => true,
-      "hosts" => "#{get_host()}",
-      "port" => "#{get_port()}"
+      "hosts" => "#{get_host_port()}"
     }
     next LogStash::Outputs::ElasticSearch.new(settings)
   end
@@ -32,7 +31,7 @@ describe "index template expected behavior", :integration => true do
     subject.receive(LogStash::Event.new("country" => "us"))
     subject.receive(LogStash::Event.new("country" => "at"))
     subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
-    subject.buffer_flush(:final => true)
+    subject.flush
     @es.indices.refresh
 
     # Wait or fail until everything's indexed.