logstash-output-elasticsearch 2.1.2-java → 2.1.4-java

data/spec/integration/outputs/update_spec.rb

@@ -9,8 +9,7 @@ describe "all protocols update actions", :integration => true do
       "manage_template" => true,
       "index" => "logstash-update",
       "template_overwrite" => true,
-      "hosts" => get_host(),
-      "port" => get_port(),
+      "hosts" => get_host_port(),
       "action" => "update"
     }
     settings['upsert'] = upsert unless upsert.nil?
@@ -47,7 +46,7 @@ describe "all protocols update actions", :integration => true do
     subject = get_es_output("456")
     subject.register
     subject.receive(LogStash::Event.new("message" => "sample message here"))
-    subject.buffer_flush(:final => true)
+    subject.flush
     expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
   end
 
@@ -55,7 +54,7 @@ describe "all protocols update actions", :integration => true do
     subject = get_es_output("123")
     subject.register
     subject.receive(LogStash::Event.new("message" => "updated message here"))
-    subject.buffer_flush(:final => true)
+    subject.flush
     r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
     insist { r["_source"]["message"] } == 'updated message here'
   end
@@ -65,7 +64,7 @@ describe "all protocols update actions", :integration => true do
       subject = get_es_output("456", '{"message": "upsert message"}')
       subject.register
       subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.buffer_flush(:final => true)
+      subject.flush
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
       insist { r["_source"]["message"] } == 'upsert message'
     end
@@ -74,7 +73,7 @@ describe "all protocols update actions", :integration => true do
       subject = get_es_output("456", nil, true)
       subject.register
       subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.buffer_flush(:final => true)
+      subject.flush
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
       insist { r["_source"]["message"] } == 'sample message here'
     end

data/spec/unit/buffer_spec.rb

@@ -0,0 +1,118 @@
+require "logstash/outputs/elasticsearch"
+require "logstash/outputs/elasticsearch/buffer"
+
+describe LogStash::Outputs::ElasticSearch::Buffer do
+  class OperationTarget # Used to track buffer flushesn
+    attr_reader :buffer, :buffer_history, :receive_count
+    def initialize
+      @buffer = nil
+      @buffer_history = []
+      @receive_count = 0
+    end
+
+    def receive(buffer)
+      @receive_count += 1
+      @buffer_history << buffer.clone
+      @buffer = buffer
+    end
+  end
+
+  let(:logger) { Cabin::Channel.get }
+  let(:max_size) { 10 }
+  let(:flush_interval) { 2 }
+  # Used to track flush count
+  let(:operation_target) { OperationTarget.new() }
+  let(:operation) { proc {|buffer| operation_target.receive(buffer) } }
+  subject(:buffer){ LogStash::Outputs::ElasticSearch::Buffer.new(logger, max_size, flush_interval, &operation) }
+
+  after(:each) do
+    buffer.stop(do_flush=false)
+  end
+
+  it "should initialize cleanly" do
+    expect(buffer).to be_a(LogStash::Outputs::ElasticSearch::Buffer)
+  end
+
+  shared_examples("a buffer with two items inside") do
+    it "should add a pushed item to the buffer" do
+      buffer.synchronize do |data|
+        expect(data).to include(item1)
+        expect(data).to include(item2)
+      end
+    end
+
+    describe "interval flushing" do
+      before do
+        sleep flush_interval + 1
+      end
+
+      it "should flush the buffer after the interval has passed" do
+        expect(operation_target.receive_count).to eql(1)
+      end
+
+      it "should clear the buffer after a successful flush" do
+        expect(operation_target.buffer).to eql([])
+      end
+    end
+
+    describe "interval flushing a stopped buffer" do
+      before do
+        buffer.stop(do_flush=false)
+        sleep flush_interval + 1
+      end
+
+      it "should not flush if the buffer is stopped" do
+        expect(operation_target.receive_count).to eql(0)
+      end
+    end
+  end
+
+  describe "with a buffer push" do
+    let(:item1) { "foo" }
+    let(:item2) { "bar" }
+
+    describe "a buffer with two items pushed to it separately" do
+      before do
+        buffer << item1
+        buffer << item2
+      end
+
+      include_examples("a buffer with two items inside")
+    end
+
+    describe "a buffer with two items pushed to it in one operation" do
+      before do
+        buffer.push_multi([item1, item2])
+      end
+
+      include_examples("a buffer with two items inside")
+    end
+  end
+
+  describe "with an empty buffer" do
+    it "should not perform an operation if the buffer is empty" do
+      buffer.flush
+      expect(operation_target.receive_count).to eql(0)
+    end
+  end
+
+  describe "flushing with an operation that raises an error" do
+    class TestError < StandardError; end
+    let(:operation) { proc {|buffer| raise TestError, "A test" } }
+    let(:item) { double("item") }
+
+    before do
+      buffer << item
+    end
+
+    it "should raise an exception" do
+      expect { buffer.flush }.to raise_error(TestError)
+    end
+
+    it "should not clear the buffer" do
+      expect do
+        buffer.flush rescue TestError
+      end.not_to change(buffer, :contents)
+    end
+  end
+end

data/spec/unit/http_client_builder_spec.rb

@@ -0,0 +1,27 @@
+require "logstash/outputs/elasticsearch"
+require "logstash/outputs/elasticsearch/http_client"
+require "logstash/outputs/elasticsearch/http_client_builder"
+
+describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
+  describe "auth setup with url encodable passwords" do
+    let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
+    let(:user) { "foo@bar"}
+    let(:password) {"baz@blah" }
+    let(:password_secured) do
+      secured = double("password")
+      allow(secured).to receive(:value).and_return(password)
+      secured
+    end
+    let(:options) { {"user" => user, "password" => password} }
+    let(:logger) { mock("logger") }
+    let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
+
+    it "should return the user verbatim" do
+      expect(auth_setup[:user]).to eql(user)
+    end
+
+    it "should return the password verbatim" do
+      expect(auth_setup[:password]).to eql(password)
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/protocol_spec.rb

@@ -2,10 +2,10 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
 require "java"
 
-describe LogStash::Outputs::Elasticsearch::HttpClient do
+describe LogStash::Outputs::ElasticSearch::HttpClient do
   describe "sniffing" do
     let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get }}
-    let(:client) { LogStash::Outputs::Elasticsearch::HttpClient.new(base_options.merge(client_opts)) }
+    let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
     let(:transport) { client.client.transport }
 
     before do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1,11 +1,9 @@
 require_relative "../../../spec/es_spec_helper"
 require "flores/random"
+require "logstash/outputs/elasticsearch"
 
 describe "outputs/elasticsearch" do
-  describe "http client create" do
-    require "logstash/outputs/elasticsearch"
-    require "elasticsearch"
-
+  context "with an active instance" do
     let(:options) {
       {
         "index" => "my-index",
@@ -21,9 +19,44 @@ describe "outputs/elasticsearch" do
     }
 
     around(:each) do |block|
-      thread = eso.register
+      eso.register
       block.call()
-      thread.kill()
+      eso.close
+    end
+
+    describe "getting a document type" do
+      it "should default to 'logs'" do
+        expect(eso.send(:get_event_type, LogStash::Event.new)).to eql("logs")
+      end
+
+      it "should get the type from the event if nothing else specified in the config" do
+        expect(eso.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
+      end
+
+      context "with 'document type set'" do
+        let(:options) { super.merge("document_type" => "bar")}
+        it "should get the event type from the 'document_type' setting" do
+          expect(eso.send(:get_event_type, LogStash::Event.new())).to eql("bar")
+        end
+      end
+
+      context "with a bad type" do
+        let(:type_arg) { ["foo"] }
+        let(:result) { eso.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
+
+        before do
+          allow(eso.instance_variable_get(:@logger)).to receive(:warn)
+          result
+        end
+
+        it "should call @logger.warn and return nil" do
+          expect(eso.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
+        end
+
+        it "should set the type to the stringified value" do
+          expect(result).to eql(type_arg.to_s)
+        end
+      end
     end
 
     describe "with path" do
@@ -62,7 +95,7 @@ describe "outputs/elasticsearch" do
   end
 
   # TODO(sissel): Improve this. I'm not a fan of using message expectations (expect().to receive...)
-  # especially with respect to logging to verify a failure/retry has occurred. For now, this 
+  # especially with respect to logging to verify a failure/retry has occurred. For now, this
   # should suffice, though.
   context "with timeout set" do
     let(:listener) { Flores::Random.tcp_listener }
@@ -76,17 +109,18 @@ describe "outputs/elasticsearch" do
       }
     end
     let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
-    
+
     before do
-      eso.register 
+      eso.register
 
       # Expect a timeout to be logged.
-      expect(eso.logger).to receive(:warn).with("Failed to flush outgoing items",
-                                                hash_including(:exception => "Manticore::SocketTimeout"))
+      expect(eso.logger).to receive(:error).with(/Attempted to send a bulk request/, anything)
     end
 
     after do
       listener[0].close
+      # Stop the receive buffer, but don't flush because that would hang forever in this case since ES never returns a result
+      eso.instance_variable_get(:@buffer).stop(false,false)
       eso.close
     end
 
@@ -94,7 +128,7 @@ describe "outputs/elasticsearch" do
       Thread.new { eso.receive(LogStash::Event.new) }
 
       # Allow the timeout to occur.
-      sleep(options["timeout"] + 0.3)
+      sleep(options["timeout"] + 0.5)
     end
   end
 end

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -27,7 +27,6 @@ describe "SSL option" do
   end
 
   context "when using ssl with client certificates" do
-
     let(:keystore_path) { Stud::Temporary.file.path }
 
     after :each do

metadata

@@ -1,39 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.1.4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-22 00:00:00.000000000 Z
+date: 2015-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: concurrent-ruby
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: elasticsearch
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.0.13
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -42,18 +33,18 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: elasticsearch
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.17
+        version: 1.0.13
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -62,135 +53,156 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.0'
+  name: stud
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: cabin
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.17
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.6'
+  name: cabin
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-core
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '0.6'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  name: logstash-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: ftw
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.42
+        version: 2.0.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.42
+  name: ftw
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.42
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: manticore
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.4.2
+  name: manticore
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: longshoreman
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: longshoreman
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: flores
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: flores
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Output events to elasticsearch
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
 - lib/logstash/outputs/elasticsearch.rb
+- lib/logstash/outputs/elasticsearch/buffer.rb
+- lib/logstash/outputs/elasticsearch/common.rb
+- lib/logstash/outputs/elasticsearch/common_configs.rb
 - lib/logstash/outputs/elasticsearch/elasticsearch-template.json
 - lib/logstash/outputs/elasticsearch/http_client.rb
+- lib/logstash/outputs/elasticsearch/http_client_builder.rb
+- lib/logstash/outputs/elasticsearch/template_manager.rb
+- logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/index_spec.rb
@@ -199,17 +211,12 @@ files:
 - spec/integration/outputs/secure_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
+- spec/unit/buffer_spec.rb
+- spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/protocol_spec.rb
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
-- spec/unit/outputs/elasticsearch/protocol_spec.rb
-- logstash-output-elasticsearch.gemspec
-- README.md
-- CHANGELOG.md
-- CONTRIBUTORS
-- Gemfile
-- LICENSE
-- NOTICE.TXT
 homepage: http://logstash.net/
 licenses:
 - apache-2.0
@@ -232,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Logstash Output to Elasticsearch
@@ -245,7 +252,9 @@ test_files:
 - spec/integration/outputs/secure_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
+- spec/unit/buffer_spec.rb
+- spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/protocol_spec.rb
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
-- spec/unit/outputs/elasticsearch/protocol_spec.rb