logstash-output-elasticsearch 2.0.0.pre.beta-java → 2.1.0-java

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -6,15 +6,19 @@ require "elasticsearch/transport/transport/http/manticore"
 
 module LogStash::Outputs::Elasticsearch
   class HttpClient
-    attr_reader :client, :options, :client_options
-    DEFAULT_OPTIONS = {
-      :port => 9200
-    }
+    attr_reader :client, :options, :client_options, :sniffer_thread
+    # This is here in case we use DEFAULT_OPTIONS in the future
+    # DEFAULT_OPTIONS = {
+    #   :setting => value
+    # }
 
     def initialize(options={})
-      @logger = Cabin::Channel.get
-      @options = DEFAULT_OPTIONS.merge(options)
+      @logger = options[:logger]
+      # Again, in case we use DEFAULT_OPTIONS in the future, uncomment this.
+      # @options = DEFAULT_OPTIONS.merge(options)
+      @options = options
       @client = build_client(@options)
+      start_sniffing!
     end
 
     def template_install(name, template, force=false)
@@ -49,26 +53,78 @@ module LogStash::Outputs::Elasticsearch
         end
       end.flatten
 
-      bulk_response = @client.bulk(:body => bulk_body)
+      @client.bulk(:body => bulk_body)
+    end
+
+    def start_sniffing!
+      if options[:sniffing]
+        @sniffer_thread = Thread.new do
+          loop do
+            sniff!
+            sleep (options[:sniffing_delay].to_f || 30)
+          end
+        end
+      end
+    end
 
-      self.class.normalize_bulk_response(bulk_response)
+    def stop_sniffing!
+      @sniffer_thread.kill() if @sniffer_thread
+    end
+
+    def sniff!
+      client.transport.reload_connections! if options[:sniffing]
+      hosts_by_name = client.transport.hosts.map {|h| h["name"]}.sort
+      @logger.debug({"count" => hosts_by_name.count, "hosts" => hosts_by_name})
+    rescue StandardError => e
+      @logger.error("Error while sniffing connection",
+                    :message => e.message,
+                    :class => e.class.name,
+                    :backtrace => e.backtrace)
     end
 
     private
 
+    # Builds a client and returns an Elasticsearch::Client
+    #
+    # The `options` is a hash where the following symbol keys have meaning:
+    #
+    # * `:hosts` - array of String. Set a list of hosts to use for communication.
+    # * `:port` - number. set the port to use to communicate with Elasticsearch
+    # * `:user` - String. The user to use for authentication.
+    # * `:password` - String. The password to use for authentication.
+    # * `:timeout` - Float. A duration value, in seconds, after which a socket
+    #    operation or request will be aborted if not yet successfull
+    # * `:client_settings` - a hash; see below for keys.
+    #
+    # The `client_settings` key is a has that can contain other settings:
+    #
+    # * `:ssl` - Boolean. Enable or disable SSL/TLS.
+    # * `:proxy` - String. Choose a HTTP HTTProxy to use.
+    # * `:path` - String. The leading path for prefixing Elasticsearch
+    #   requests. This is sometimes used if you are proxying Elasticsearch access
+    #   through a special http path, such as using mod_rewrite.
     def build_client(options)
-      uris = options[:hosts].map do |host|
-        "http://#{host}:#{options[:port]}#{options[:client_settings][:path]}"
+      hosts = options[:hosts] || ["127.0.0.1"]
+      client_settings = options[:client_settings] || {}
+      timeout = options[:timeout] || 0
+
+      uris = hosts.map do |host|
+        proto = client_settings[:ssl] ? "https"  : "http"
+        if host =~ /:\d+\z/
+          "#{proto}://#{host}#{client_settings[:path]}"
+        else
+          # Use default port of 9200 if none provided with host.
+          "#{proto}://#{host}:9200#{client_settings[:path]}"
+        end
       end
 
       @client_options = {
         :hosts => uris,
-        :ssl => options[:client_settings][:ssl],
-        :reload_connections => options[:client_settings][:reload_connections],
-        :transport_options => {  # manticore settings so we
-          :socket_timeout => 0,  # do not timeout socket reads
-          :request_timeout => 0,  # and requests
-          :proxy => options[:client_settings][:proxy]
+        :ssl => client_settings[:ssl],
+        :transport_options => {
+          :socket_timeout => timeout,
+          :request_timeout => timeout,
+          :proxy => client_settings[:proxy]
         },
         :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore
       }
@@ -78,27 +134,9 @@ module LogStash::Outputs::Elasticsearch
         @client_options[:headers] = { "Authorization" => "Basic #{token}" }
       end
 
-      c = Elasticsearch::Client.new(client_options)
-      c.transport.reload_connections! if options[:client_settings][:reload_connections]
-      c
-    end
+      @logger.debug? && @logger.debug("Elasticsearch HTTP client options", client_options)
 
-    def self.normalize_bulk_response(bulk_response)
-      if bulk_response["errors"]
-        # The structure of the response from the REST Bulk API is follows:
-        # {"took"=>74, "errors"=>true, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
-        #                                                    "_type"=>"logs",
-        #                                                    "_id"=>"AUxTS2C55Jrgi-hC6rQF",
-        #                                                    "_version"=>1,
-        #                                                    "status"=>400,
-        #                                                    "error"=>"MapperParsingException[failed to parse]..."}}]}
-        # where each `item` is a hash of {OPTYPE => Hash[]}. calling first, will retrieve
-        # this hash as a single array with two elements, where the value is the second element (i.first[1])
-        # then the status of that item is retrieved.
-        {"errors" => true, "statuses" => bulk_response["items"].map { |i| i.first[1]['status'] }}
-      else
-        {"errors" => false}
-      end
+      Elasticsearch::Client.new(client_options)
     end
 
     def template_exists?(name)

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '2.0.0-beta'
+  s.version         = '2.1.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -21,13 +21,13 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency 'concurrent-ruby'
-  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.10', '~> 1.0']
+  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.13', '~> 1.0']
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency 'cabin', ['~> 0.6']
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0.snapshot", "< 3.0.0"
 
   s.add_development_dependency 'ftw', '~> 0.0.42'
-  s.add_development_dependency 'logstash-input-generator'
+  s.add_development_dependency 'logstash-codec-plain'
 
   if RUBY_PLATFORM == 'java'
     s.platform = RUBY_PLATFORM
@@ -36,4 +36,5 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'longshoreman'
+  s.add_development_dependency 'flores'
 end

data/spec/es_spec_helper.rb

@@ -4,6 +4,7 @@ require "logstash/plugin"
 require "logstash/json"
 require "stud/try"
 require "longshoreman"
+require "logstash/outputs/elasticsearch"
 
 CONTAINER_NAME = "logstash-output-elasticsearch-#{rand(999).to_s}"
 CONTAINER_IMAGE = "elasticsearch"

data/spec/integration/outputs/index_spec.rb

@@ -6,13 +6,16 @@ shared_examples "an indexer" do
     let(:event_count) { 10000 + rand(500) }
     let(:flush_size) { rand(200) + 1 }
     let(:config) { "not implemented" }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-    it "ships events" do
-      insist { config } != "not implemented"
-
-      pipeline = LogStash::Pipeline.new(config)
-      pipeline.run
+    before do
+      subject.register
+      event_count.times do
+        subject.receive(LogStash::Event.new("message" => "Hello World!", "type" => type))
+      end
+    end
 
+    it "ships events" do
       index_url = "http://#{get_host}:#{get_port}/#{index}"
 
       ftw = FTW::Agent.new
@@ -42,23 +45,12 @@ end
 describe "an indexer with custom index_type", :integration => true do
   it_behaves_like "an indexer" do
     let(:config) {
-      <<-CONFIG
-      input {
-        generator {
-          message => "hello world"
-          count => #{event_count}
-          type => "#{type}"
-        }
-      }
-      output {
-        elasticsearch {
-          hosts => "#{get_host()}"
-          port => "#{get_port}"
-          index => "#{index}"
-          flush_size => #{flush_size}
-        }
+      {
+        "hosts" => get_host,
+        "port" => get_port,
+        "index" => index,
+        "flush_size" => flush_size
       }
-      CONFIG
     }
   end
 end
@@ -67,22 +59,12 @@ describe "an indexer with no type value set (default to logs)", :integration =>
   it_behaves_like "an indexer" do
     let(:type) { "logs" }
     let(:config) {
-      <<-CONFIG
-      input {
-        generator {
-          message => "hello world"
-          count => #{event_count}
-        }
-      }
-      output {
-        elasticsearch {
-          hosts => "#{get_host()}"
-          port => "#{get_port}"
-          index => "#{index}"
-          flush_size => #{flush_size}
-        }
+      {
+        "hosts" => get_host,
+        "port" => get_port,
+        "index" => index,
+        "flush_size" => flush_size
       }
-      CONFIG
     }
   end
 end

data/spec/integration/outputs/retry_spec.rb

@@ -11,8 +11,18 @@ describe "failures in bulk class expected behavior", :integration => true do
   let(:max_retries) { 3 }
 
   def mock_actions_with_response(*resp)
-    LogStash::Outputs::Elasticsearch::HttpClient
-      .any_instance.stub(:bulk).and_return(*resp)
+    expanded_responses = resp.map do |resp|
+      items = resp["statuses"] && resp["statuses"].map do |status|
+        {"create" => {"status" => status, "error" => "Error for #{status}"}}
+      end
+
+      {
+        "errors" => resp["errors"],
+        "items" => items
+      }
+    end
+
+    allow_any_instance_of(LogStash::Outputs::Elasticsearch::HttpClient).to receive(:bulk).and_return(*expanded_responses)
   end
 
   subject! do
@@ -61,7 +71,7 @@ describe "failures in bulk class expected behavior", :integration => true do
     end
     subject.register
     subject.receive(event1)
-    subject.teardown
+    subject.close
   end
 
   it "should retry actions with response status of 503" do
@@ -109,7 +119,7 @@ describe "failures in bulk class expected behavior", :integration => true do
     subject.register
     subject.receive(invalid_event)
     expect(subject).not_to receive(:retry_push)
-    subject.teardown
+    subject.close
 
     @es.indices.refresh
     sleep(5)
@@ -123,7 +133,7 @@ describe "failures in bulk class expected behavior", :integration => true do
     subject.register
     subject.receive(event1)
     expect(subject).not_to receive(:retry_push)
-    subject.teardown
+    subject.close
     @es.indices.refresh
     sleep(5)
     Stud::try(10.times) do
@@ -136,7 +146,7 @@ describe "failures in bulk class expected behavior", :integration => true do
     subject.register
     subject.receive(invalid_event)
     subject.receive(event1)
-    subject.teardown
+    subject.close
 
     @es.indices.refresh
     sleep(5)

data/spec/integration/outputs/routing_spec.rb

@@ -7,14 +7,17 @@ shared_examples "a routing indexer" do
     let(:flush_size) { rand(200) + 1 }
     let(:routing) { "not_implemented" }
     let(:config) { "not_implemented" }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-    it "ships events" do
-      insist { routing } != "not_implemented"
-      insist { config } != "not_implemented"
+    before do
+      subject.register
+      event_count.times do
+        subject.receive(LogStash::Event.new("message" => "Hello World!", "type" => type))
+      end
+    end
 
-      pipeline = LogStash::Pipeline.new(config)
-      pipeline.run
 
+    it "ships events" do
       index_url = "http://#{get_host()}:#{get_port()}/#{index}"
 
       ftw = FTW::Agent.new
@@ -36,24 +39,13 @@ describe "(http protocol) index events with static routing", :integration => tru
   it_behaves_like 'a routing indexer' do
     let(:routing) { "test" }
     let(:config) {
-      <<-CONFIG
-      input {
-        generator {
-          message => "hello world"
-          count => #{event_count}
-          type => "#{type}"
-        }
-      }
-      output {
-        elasticsearch {
-          hosts => "#{get_host()}"
-          port => "#{get_port()}"
-          index => "#{index}"
-          flush_size => #{flush_size}
-          routing => "#{routing}"
-        }
+      {
+        "hosts" => get_host,
+        "port" => get_port,
+        "index" => index,
+        "flush_size" => flush_size,
+        "routing" => routing
       }
-      CONFIG
     }
   end
 end
@@ -62,24 +54,13 @@ describe "(http_protocol) index events with fieldref in routing value", :integra
   it_behaves_like 'a routing indexer' do
     let(:routing) { "test" }
     let(:config) {
-      <<-CONFIG
-      input {
-        generator {
-          message => "#{routing}"
-          count => #{event_count}
-          type => "#{type}"
-        }
-      }
-      output {
-        elasticsearch {
-          hosts => "#{get_host()}"
-          port => "#{get_port()}"
-          index => "#{index}"
-          flush_size => #{flush_size}
-          routing => "%{message}"
-        }
+      {
+        "hosts" => get_host,
+        "port" => get_port,
+        "index" => index,
+        "flush_size" => flush_size,
+        "routing" => "%{message}"
       }
-      CONFIG
     }
   end
 end

data/spec/unit/outputs/elasticsearch/protocol_spec.rb

@@ -3,24 +3,39 @@ require "logstash/outputs/elasticsearch/http_client"
 require "java"
 
 describe LogStash::Outputs::Elasticsearch::HttpClient do
-  context "successful" do
-    it "should map correctly" do
-      bulk_response = {"took"=>74, "errors"=>false, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
-                                                                          "_type"=>"logs", "_id"=>"AUxTS2C55Jrgi-hC6rQF",
-                                                                          "_version"=>1, "status"=>201}}]} 
-      actual = LogStash::Outputs::Elasticsearch::HttpClient.normalize_bulk_response(bulk_response)
-      insist { actual } == {"errors"=> false}
+  describe "sniffing" do
+    let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get }}
+    let(:client) { LogStash::Outputs::Elasticsearch::HttpClient.new(base_options.merge(client_opts)) }
+    let(:transport) { client.client.transport }
+
+    before do
+      allow(transport).to receive(:reload_connections!)
     end
-  end
 
-  context "contains failures" do
-    it "should map correctly" do
-      bulk_response = {"took"=>71, "errors"=>true,
-                       "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
-                                             "_type"=>"logs", "_id"=>"AUxTQ_OI5Jrgi-hC6rQB", "status"=>400,
-                                             "error"=>"MapperParsingException[failed to parse]..."}}]}
-      actual = LogStash::Outputs::Elasticsearch::HttpClient.normalize_bulk_response(bulk_response)
-      insist { actual } == {"errors"=> true, "statuses"=> [400]}
+    context "with sniffing enabled" do
+      let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
+
+      after do
+        client.stop_sniffing!
+      end
+
+      it "should start the sniffer" do
+        expect(client.sniffer_thread).to be_a(Thread)
+      end
+
+      it "should periodically sniff the client" do
+        sleep 2
+        expect(transport).to have_received(:reload_connections!)
+      end
     end
+
+    context "with sniffing disabled" do
+      let(:client_opts) { {:sniffing => false} }
+
+      it "should not start the sniffer" do
+        expect(client.sniffer_thread).to be_nil
+      end
+    end
+
   end
 end