logstash-output-elasticsearch 7.4.3-java → 8.0.0-java

data/spec/integration/outputs/update_spec.rb

@@ -1,116 +1,114 @@
 require_relative "../../../spec/es_spec_helper"
 
-if ESHelper.es_version_satisfies?(">= 2")
-  describe "Update actions without scripts", :integration => true do
-    require "logstash/outputs/elasticsearch"
-
-    def get_es_output( options={} )
-      settings = {
-        "manage_template" => true,
-        "index" => "logstash-update",
-        "template_overwrite" => true,
-        "hosts" => get_host_port(),
-        "action" => "update"
-      }
-      LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+describe "Update actions without scripts", :integration => true, :version_greater_than_equal_to_2x => true do
+  require "logstash/outputs/elasticsearch"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "update"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => 'logs',
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
+    @es.indices.refresh
+  end
+
+  it "should fail without a document_id" do
+    subject = get_es_output
+    expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+  end
+
+  context "when update only" do
+    it "should not create new document" do
+      subject = get_es_output({ 'document_id' => "456" } )
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+    end
+
+    it "should update existing document" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["message"] } == 'updated message here'
+    end
+
+    # The es ruby client treats the data field differently. Make sure this doesn't
+    # raise an exception
+    it "should update an existing document that has a 'data' field" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["data"] } == 'updated message here'
+      insist { r["_source"]["message"] } == 'foo'
+    end
+
+    it "should allow default (internal) version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+      subject.register
     end
 
-    before :each do
-      @es = get_client
-      # Delete all templates first.
-      # Clean ES of data before we start.
-      @es.indices.delete_template(:name => "*")
-      # This can fail if there are no indexes, ignore failure.
-      @es.indices.delete(:index => "*") rescue nil
-      @es.index(
-        :index => 'logstash-update',
-        :type => 'logs',
-        :id => "123",
-        :body => { :message => 'Test', :counter => 1 }
-      )
-      @es.indices.refresh
+    it "should allow internal version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+      subject.register
     end
 
-    it "should fail without a document_id" do
-      subject = get_es_output
+    it "should not allow external version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
-    context "when update only" do
-      it "should not create new document" do
-        subject = get_es_output({ 'document_id' => "456" } )
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-      end
-
-      it "should update existing document" do
-        subject = get_es_output({ 'document_id' => "123" })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-        insist { r["_source"]["message"] } == 'updated message here'
-      end
-
-      # The es ruby client treats the data field differently. Make sure this doesn't
-      # raise an exception
-      it "should update an existing document that has a 'data' field" do
-        subject = get_es_output({ 'document_id' => "123" })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-        insist { r["_source"]["data"] } == 'updated message here'
-        insist { r["_source"]["message"] } == 'foo'
-      end
-
-      it "should allow default (internal) version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99" })
-        subject.register
-      end
-
-      it "should allow internal version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
-        subject.register
-      end
-
-      it "should not allow external version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
-
-      it "should not allow external_gt version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
-
-      it "should not allow external_gte version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
+    it "should not allow external_gt version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
 
+    it "should not allow external_gte version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
-    context "when update with upsert" do
-      it "should create new documents with provided upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-        insist { r["_source"]["message"] } == 'upsert message'
-      end
-
-      it "should create new documents with event/doc as upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-        insist { r["_source"]["message"] } == 'sample message here'
-      end
-
-      it "should fail on documents with event/doc as upsert at external version" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
+  end
+
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'upsert message'
+    end
+
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'sample message here'
+    end
+
+    it "should fail on documents with event/doc as upsert at external version" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end
 end

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -44,44 +44,19 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
     end
   end
 
-  describe "bad response codes" do
-    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
-
-    it "should raise a bad response code error" do
-      resp = double("response")
-      allow(resp).to receive(:call)
-      allow(resp).to receive(:code).and_return(500)
-      allow(resp).to receive(:body).and_return("a body")
-
-      expect(subject.manticore).to receive(:get).
-        with(uri.to_s + "/", anything).
-        and_return(resp)
-
-      uri_with_path = uri.clone
-      uri_with_path.path = "/"
-
-      expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
-        with(resp.code, uri_with_path, nil, resp.body).and_call_original
-
-      expect do
-        subject.perform_request(uri, :get, "/")
-      end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
-    end
-  end
-
   describe "format_url" do
     let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
     let(:path) { "_bulk" }
     subject { described_class.new(double("logger"), {}) }
 
     it "should add the path argument to the uri's path" do
-      expect(subject.format_url(url, path).path).to eq("/path/_bulk")
+      expect(java.net.URI.new(subject.format_url(url, path)).path).to eq("/path/_bulk")
     end
 
     context "when uri contains query parameters" do
       let(:query_params) { "query=value&key=value2" }
       let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
-      let(:formatted) { subject.format_url(url, path)}
+      let(:formatted) { java.net.URI.new(subject.format_url(url, path))}
 
       it "should retain query_params after format" do
         expect(formatted.query).to eq(query_params)
@@ -98,7 +73,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
     
     context "when the path contains query parameters" do
       let(:path) { "/special_bulk?pathParam=1"}
-      let(:formatted) { subject.format_url(url, path) }
+      let(:formatted) { java.net.URI.new(subject.format_url(url, path)) }
       
       it "should add the path correctly" do
         expect(formatted.path).to eq("#{url.path}special_bulk")
@@ -111,10 +86,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
 
     context "when uri contains credentials" do
       let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
-      let(:formatted) { subject.format_url(url, path) }
+      let(:formatted) { java.net.URI.new(subject.format_url(url, path)) }
 
       it "should remove credentials after format" do
-        expect(formatted.userinfo).to be_nil
+        expect(formatted.user_info).to be_nil
       end
     end
   end

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -21,9 +21,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
-    allow(subject).to receive(:connected_es_versions).with(any_args).and_return(["0.0.0"])
-    allow(subject).to receive(:get_es_version).with(any_args).and_return("0.0.0")
-
     subject.start
   end
 

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -29,7 +29,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     let(:http_hostname_port) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}") }
     let(:https_hostname_port) { ::LogStash::Util::SafeURI.new("https://#{hostname_port}") }
     let(:http_hostname_port_path) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}/path") }
-
+    
     shared_examples("proper host handling") do
       it "should properly transform a host:port string to a URL" do
         expect(subject.host_to_url(hostname_port_uri).to_s).to eq(http_hostname_port.to_s + "/")
@@ -58,7 +58,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         context "when SSL is false" do
           let(:ssl) { false }
           let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
-
+          
           it "should refuse to handle an https url" do
             expect {
               subject.host_to_url(https_hostname_port)
@@ -72,32 +72,32 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
             subject
             expect(subject.host_to_url(https_hostname_port).to_s).to eq(https_hostname_port.to_s + "/")
           end
-        end
+        end       
       end
 
       describe "path" do
         let(:url) { http_hostname_port_path }
         let(:base_options) { super.merge(:hosts => [url]) }
-
+        
         it "should allow paths in a url" do
           expect(subject.host_to_url(url)).to eq(url)
         end
 
         context "with the path option set" do
           let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) }
-
+          
           it "should not allow paths in two places" do
             expect {
               subject.host_to_url(url)
             }.to raise_error(LogStash::ConfigurationError)
           end
         end
-
+        
         context "with a path missing a leading /" do
           let(:url) { http_hostname_port }
           let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) }
-
-
+          
+          
           it "should automatically insert a / in front of path overlays" do
             expected = url.clone
             expected.path = url.path + "/otherpath"
@@ -132,7 +132,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     }
 
     it "returns the hash response" do
-      expect(subject.pool).to receive(:get).with(path, nil).and_return(get_response)
+      expect(subject.pool).to receive(:get).with(path, nil).and_return([nil, get_response])
       expect(subject.get(path)["body"]).to eq(body)
     end
   end
@@ -182,6 +182,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
   describe "#bulk" do
     subject { described_class.new(base_options) }
+
+    require "json"
     let(:message) { "hey" }
     let(:actions) { [
       ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
@@ -191,7 +193,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
       let(:message) { "a" * (target_bulk_bytes + 1) }
 
-      it "sends the message as its own bulk payload" do
+      it "should be handled properly" do
         allow(subject).to receive(:join_bulk_responses)
         expect(subject).to receive(:bulk_send).once do |data|
           expect(data.size).to be > target_bulk_bytes
@@ -207,7 +209,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
         ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
       ]}
-
       it "executes one bulk_send operation" do
         allow(subject).to receive(:join_bulk_responses)
         expect(subject).to receive(:bulk_send).once

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -6,21 +6,32 @@ require "json"
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
   describe ".get_es_major_version" do
+    let(:es_1x_version) { '{ "number" : "1.7.0", "build_hash" : "929b9739cae115e73c346cb5f9a6f24ba735a743", "build_timestamp" : "2015-07-16T14:31:07Z", "build_snapshot" : false, "lucene_version" : "4.10.4" }' }
+    let(:es_2x_version) { '{ "number" : "2.3.4", "build_hash" : "e455fd0c13dceca8dbbdbb1665d068ae55dabe3f", "build_timestamp" : "2016-06-30T11:24:31Z", "build_snapshot" : false, "lucene_version" : "5.5.0" }' }
+    let(:es_5x_version) { '{ "number" : "5.0.0-alpha4", "build_hash" : "b0da471", "build_date" : "2016-06-22T12:33:48.164Z", "build_snapshot" : false, "lucene_version" : "6.1.0" }' }
     let(:client) { double("client") }
-
-    before(:each) do
-      allow(client).to receive(:connected_es_versions).and_return(["5.3.0"])
+    context "elasticsearch 1.x" do
+      before(:each) do
+        allow(client).to receive(:get_version).and_return(JSON.parse(es_1x_version))
+      end
+      it "detects major version is 1" do
+        expect(described_class.get_es_major_version(client)).to eq("1")
+      end
     end
-
-    it "picks the largest major version" do
-      expect(described_class.get_es_major_version(client)).to eq(5)
+    context "elasticsearch 2.x" do
+      before(:each) do
+        allow(client).to receive(:get_version).and_return(JSON.parse(es_2x_version))
+      end
+      it "detects major version is 2" do
+        expect(described_class.get_es_major_version(client)).to eq("2")
+      end
     end
-    context "if there are nodes with multiple major versions" do
+    context "elasticsearch 5.x" do
       before(:each) do
-        allow(client).to receive(:connected_es_versions).and_return(["5.3.0", "6.0.0"])
+        allow(client).to receive(:get_version).and_return(JSON.parse(es_5x_version))
       end
-      it "picks the largest major version" do
-        expect(described_class.get_es_major_version(client)).to eq(6)
+      it "detects major version is 5" do
+        expect(described_class.get_es_major_version(client)).to eq("5")
       end
     end
   end
@@ -28,18 +39,19 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
   describe ".default_template_path" do
     context "elasticsearch 1.x" do
       it "chooses the 2x template" do
-        expect(described_class.default_template_path(1)).to match(/elasticsearch-template-es2x.json/)
+        expect(described_class.default_template_path("1")).to match(/elasticsearch-template-es2x.json/)
       end
     end
     context "elasticsearch 2.x" do
       it "chooses the 2x template" do
-        expect(described_class.default_template_path(2)).to match(/elasticsearch-template-es2x.json/)
+        expect(described_class.default_template_path("2")).to match(/elasticsearch-template-es2x.json/)
       end
     end
     context "elasticsearch 5.x" do
       it "chooses the 5x template" do
-        expect(described_class.default_template_path(5)).to match(/elasticsearch-template-es5x.json/)
+        expect(described_class.default_template_path("5")).to match(/elasticsearch-template-es5x.json/)
       end
     end
   end
+
 end