logstash-output-elasticsearch 3.0.2-java → 4.1.0-java

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -1,6 +1,5 @@
 require_relative "../../../spec/es_spec_helper"
 require 'stud/temporary'
-require 'elasticsearch'
 require "logstash/outputs/elasticsearch"
 
 describe "Proxy option" do
@@ -15,7 +14,7 @@ describe "Proxy option" do
   }
 
   before do
-    allow(::Elasticsearch::Client).to receive(:new).with(any_args)
+    allow(::Manticore::Client).to receive(:new).with(any_args)
   end
 
   describe "valid configs" do
@@ -27,8 +26,8 @@ describe "Proxy option" do
       let(:proxy) { "http://127.0.0.1:1234" }
 
       it "should set the proxy to the exact value" do
-        expect(::Elasticsearch::Client).to have_received(:new) do |options|
-          expect(options[:transport_options][:proxy]).to eql(proxy)
+        expect(::Manticore::Client).to have_received(:new) do |options|
+          expect(options[:proxy]).to eql(proxy)
         end
       end
     end
@@ -38,8 +37,8 @@ describe "Proxy option" do
 
       it "should pass through the proxy values as symbols" do
         expected = {:hosts => proxy["hosts"], :protocol => proxy["protocol"]}
-        expect(::Elasticsearch::Client).to have_received(:new) do |options|
-          expect(options[:transport_options][:proxy]).to eql(expected)
+        expect(::Manticore::Client).to have_received(:new) do |options|
+          expect(options[:proxy]).to eql(expected)
         end
       end
     end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -14,9 +14,8 @@ describe "outputs/elasticsearch" do
 
     let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
 
-    let(:manticore_host) {
-      eso.client.send(:client).transport.options[:hosts].first
-    }
+    let(:manticore_urls) { eso.client.pool.urls }
+    let(:manticore_url) { manticore_urls.first }
 
     let(:do_register) { true }
 
@@ -67,17 +66,15 @@ describe "outputs/elasticsearch" do
       end
 
       it "should properly set the path on the HTTP client adding slashes" do
-        expect(manticore_host).to include("/" + options["path"] + "/")
+        expect(manticore_url.path).to eql("/" + options["path"] + "/")
       end
 
       context "with extra slashes" do
         let(:path) { "/slashed-path/ "}
-        let(:eso) {
-          LogStash::Outputs::ElasticSearch.new(options.merge("path" => "/some-path/"))
-        }
+        let(:options) { super.merge("path" => "/some-path/") }
 
         it "should properly set the path on the HTTP client without adding slashes" do
-          expect(manticore_host).to include(options["path"])
+          expect(manticore_url.path).to eql(options["path"])
         end
       end
 
@@ -88,7 +85,7 @@ describe "outputs/elasticsearch" do
           o["hosts"] = ["http://localhost:9200/mypath/"]
           o
         end
-        let(:client_host_path) { URI.parse(eso.client.client_options[:hosts].first).path }
+        let(:client_host_path) { manticore_url.path }
 
         it "should initialize without error" do
           expect { eso }.not_to raise_error
@@ -133,15 +130,12 @@ describe "outputs/elasticsearch" do
     end
     describe "without a port specified" do
       it "should properly set the default port (9200) on the HTTP client" do
-        expect(manticore_host).to include("9200")
+        expect(manticore_url.port).to eql(9200)
       end
     end
     describe "with a port other than 9200 specified" do
-      let(:manticore_host) {
-        eso.client.send(:client).transport.options[:hosts].last
-      }
       it "should properly set the specified port on the HTTP client" do
-        expect(manticore_host).to include("9202")
+        expect(manticore_urls.any? {|u| u.port == 9202}).to eql(true)
       end
     end
 
@@ -168,9 +162,6 @@ describe "outputs/elasticsearch" do
 
   end
 
-  # TODO(sissel): Improve this. I'm not a fan of using message expectations (expect().to receive...)
-  # especially with respect to logging to verify a failure/retry has occurred. For now, this
-  # should suffice, though.
   context "with timeout set" do
     let(:listener) { Flores::Random.tcp_listener }
     let(:port) { listener[2] }
@@ -188,21 +179,15 @@ describe "outputs/elasticsearch" do
       eso.register
 
       # Expect a timeout to be logged.
-      expect(eso.logger).to receive(:error).with(/Attempted to send a bulk request/, anything)
-    end
-
-    after do
-      listener[0].close
-      # Stop the receive buffer, but don't flush because that would hang forever in this case since ES never returns a result
-      eso.instance_variable_get(:@buffer).stop(false,false)
-      eso.close
+      expect(eso.logger).to receive(:error).with(/Attempted to send a bulk request to Elasticsearch/i, anything).at_least(:once)
+      expect(eso.client).to receive(:bulk).at_least(:twice).and_call_original
     end
 
     it "should fail after the timeout" do
-      Thread.new { eso.receive(LogStash::Event.new) }
+      Thread.new { eso.multi_receive([LogStash::Event.new]) }
 
-      # Allow the timeout to occur.
-      sleep(options["timeout"] + 0.5)
+      # Allow the timeout to occur
+      sleep 6
     end
   end
 
@@ -230,12 +215,12 @@ describe "outputs/elasticsearch" do
   describe "SSL end to end" do
     shared_examples("an encrypted client connection") do
       it "should enable SSL in manticore" do
-        expect(eso.client.client_options[:hosts].map {|h| URI.parse(h).scheme}.uniq).to eql(['https'])
+        expect(eso.client.pool.urls.map(&:scheme).uniq).to eql(['https'])
       end
     end
 
     let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
-    subject(:manticore) { eso.client.client}
+    subject(:manticore) { eso.client.pool.adapter.client}
 
     before do
       eso.register
@@ -276,4 +261,22 @@ describe "outputs/elasticsearch" do
       end
     end
   end
+
+  describe "sleep interval calculation" do
+    let(:retry_max_interval) { 64 }
+    subject(:eso) { LogStash::Outputs::ElasticSearch.new("retry_max_interval" => retry_max_interval) }
+
+    it "should double the given value" do
+      expect(eso.next_sleep_interval(2)).to eql(4)
+      expect(eso.next_sleep_interval(32)).to eql(64)
+    end
+
+    it "should not increase the value past the max retry interval" do
+      sleep_interval = 2
+      100.times do
+        sleep_interval = eso.next_sleep_interval(sleep_interval)
+        expect(sleep_interval).to be <= retry_max_interval
+      end
+    end
+  end
 end

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -6,22 +6,26 @@ describe "SSL option" do
     subject do
       require "logstash/outputs/elasticsearch"
       settings = {
-        "hosts" => "node01",
+        "hosts" => "localhost",
         "ssl" => true,
-        "ssl_certificate_verification" => false
+        "ssl_certificate_verification" => false,
+        "pool_max" => 1,
+        "pool_max_per_route" => 1
       }
       next LogStash::Outputs::ElasticSearch.new(settings)
     end
 
     it "should pass the flag to the ES client" do
-      expect(::Elasticsearch::Client).to receive(:new) do |args|
+      expect(::Manticore::Client).to receive(:new) do |args|
         expect(args[:ssl]).to eq(:enabled => true, :verify => false)
       end
       subject.register
     end
 
-    it "print a warning" do
-      expect(subject.logger).to receive(:warn)
+    it "should print a warning" do
+      disabled_matcher = /You have enabled encryption but DISABLED certificate verification/
+      expect(subject.logger).to receive(:warn).with(disabled_matcher).at_least(:once)
+      allow(subject.logger).to receive(:warn).with(any_args)
       subject.register
     end
   end
@@ -45,7 +49,7 @@ describe "SSL option" do
     end
 
     it "should pass the keystore parameters to the ES client" do
-      expect(::Elasticsearch::Client).to receive(:new) do |args|
+      expect(::Manticore::Client).to receive(:new) do |args|
         expect(args[:ssl]).to include(:keystore => keystore_path, :keystore_password => "test")
       end
       subject.register

metadata

@@ -1,187 +1,167 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 4.1.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-10 00:00:00.000000000 Z
+date: 2016-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :runtime
-- !ruby/object:Gem::Dependency
-  name: elasticsearch
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.13
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.13
-    - - ~>
+        version: 0.0.17
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.0'
+  name: stud
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.17
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 0.0.17
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '0.6'
+  name: cabin
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: cabin
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '2.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-  prerelease: false
-  type: :runtime
-- !ruby/object:Gem::Dependency
+        version: 0.0.42
   name: ftw
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.42
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.42
+        version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :development
-- !ruby/object:Gem::Dependency
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
   name: manticore
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.4
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
-    - - <
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  prerelease: false
-  type: :runtime
-- !ruby/object:Gem::Dependency
+        version: '0'
   name: logstash-devutils
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: longshoreman
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: longshoreman
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: flores
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: flores
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: elasticsearch
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -195,15 +175,17 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/outputs/elasticsearch.rb
-- lib/logstash/outputs/elasticsearch/buffer.rb
 - lib/logstash/outputs/elasticsearch/common.rb
 - lib/logstash/outputs/elasticsearch/common_configs.rb
 - lib/logstash/outputs/elasticsearch/elasticsearch-template.json
 - lib/logstash/outputs/elasticsearch/http_client.rb
+- lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb
+- lib/logstash/outputs/elasticsearch/http_client/pool.rb
 - lib/logstash/outputs/elasticsearch/http_client_builder.rb
 - lib/logstash/outputs/elasticsearch/template_manager.rb
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
+- spec/fixtures/5x_node_resp.json
 - spec/fixtures/scripts/scripted_update.groovy
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
@@ -216,8 +198,9 @@ files:
 - spec/integration/outputs/secure_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
-- spec/unit/buffer_spec.rb
 - spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb
+- spec/unit/outputs/elasticsearch/http_client/pool_spec.rb
 - spec/unit/outputs/elasticsearch/http_client_spec.rb
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb
@@ -234,12 +217,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
@@ -250,6 +233,7 @@ specification_version: 4
 summary: Logstash Output to Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
+- spec/fixtures/5x_node_resp.json
 - spec/fixtures/scripts/scripted_update.groovy
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
@@ -262,8 +246,9 @@ test_files:
 - spec/integration/outputs/secure_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
-- spec/unit/buffer_spec.rb
 - spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb
+- spec/unit/outputs/elasticsearch/http_client/pool_spec.rb
 - spec/unit/outputs/elasticsearch/http_client_spec.rb
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb