logstash-output-elasticsearch-test 11.16.0-x86_64-linux

data/spec/integration/outputs/index_spec.rb

@@ -0,0 +1,421 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+require 'cgi'
+
+describe "TARGET_BULK_BYTES", :integration => true do
+  let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+  let(:event_count) { 1000 }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index
+      }
+  }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+  before do
+    subject.register
+    allow(subject.client).to receive(:bulk_send).with(any_args).and_call_original
+    subject.multi_receive(events)
+  end
+
+  describe "batches that are too large for one" do
+    let(:event) { LogStash::Event.new("message" => "a " * (((target_bulk_bytes/2) / event_count)+1)) }
+
+    it "should send in two batches" do
+      expect(subject.client).to have_received(:bulk_send).twice do |payload|
+        expect(payload.size).to be <= target_bulk_bytes
+      end
+    end
+
+    describe "batches that fit in one" do
+      # Normally you'd want to generate a request that's just 1 byte below the limit, but it's
+      # impossible to know how many bytes an event will serialize as with bulk proto overhead
+      let(:event) { LogStash::Event.new("message" => "a") }
+
+      it "should send in one batch" do
+        expect(subject.client).to have_received(:bulk_send).once do |payload|
+          expect(payload.size).to be <= target_bulk_bytes
+        end
+      end
+    end
+  end
+end
+
+def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false); require 'open3'
+  cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
+  begin
+    out, err, status = Open3.capture3(cmd)
+  rescue Errno::ENOENT
+    fail "curl not available, make sure curl binary is installed and available on $PATH"
+  end
+
+  if status.success?
+    http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+
+    if http_status.strip[0].to_i > 2
+      error = (LogStash::Json.load(out)['error']) rescue nil
+      if error
+        if retrieve_err_payload
+          return error
+        else
+          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
+        end
+      else
+        warn out
+        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+      end
+    end
+
+    LogStash::Json.load(out)
+  else
+    warn out
+    fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+  end
+end
+
+describe "indexing with sprintf resolution", :integration => true do
+  let(:message) { "Hello from #{__FILE__}" }
+  let(:event) { LogStash::Event.new("message" => message, "type" => type) }
+  let (:index) { "%{[index_name]}_dynamic" }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+  let(:event_count) { 1 }
+  let(:user) { "simpleuser" }
+  let(:password) { "abc123" }
+  let(:config) do
+    {
+      "hosts" => [ get_host_port ],
+      "user" => user,
+      "password" => password,
+      "index" => index
+    }
+  end
+  let(:events) { event_count.times.map { event }.to_a }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) { "#{es_url}/#{index}" }
+
+  let(:curl_opts) { nil }
+
+  let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
+  let(:es_admin_pass) { 'elastic' }
+
+  let(:initial_events) { [] }
+
+  let(:do_register) { true }
+
+  before do
+    subject.register if do_register
+    subject.multi_receive(initial_events) if initial_events
+  end
+
+  after do
+    subject.do_close
+  end
+
+  let(:event) { LogStash::Event.new("message" => message, "type" => type, "index_name" => "test") }
+
+  it "should index successfully when field is resolved" do
+    expected_index_name = "test_dynamic"
+    subject.multi_receive(events)
+
+#     curl_and_get_json_response "#{es_url}/_refresh", method: :post
+
+    result = curl_and_get_json_response "#{es_url}/#{expected_index_name}"
+
+    expect(result[expected_index_name]).not_to be(nil)
+  end
+
+  context "when dynamic field doesn't resolve the index_name" do
+    let(:event) { LogStash::Event.new("message" => message, "type" => type) }
+    let(:dlq_writer) { double('DLQ writer') }
+    before { subject.instance_variable_set('@dlq_writer', dlq_writer) }
+
+    it "should doesn't create an index name with unresolved placeholders" do
+      expect(dlq_writer).to receive(:write).once.with(event, a_string_including("Badly formatted index, after interpolation still contains placeholder"))
+      subject.multi_receive(events)
+
+      escaped_index_name = CGI.escape("%{[index_name]}_dynamic")
+      result = curl_and_get_json_response "#{es_url}/#{escaped_index_name}", retrieve_err_payload: true
+      expect(result["root_cause"].first()["type"]).to eq("index_not_found_exception")
+    end
+  end
+end
+
+describe "indexing" do
+  let(:message) { "Hello from #{__FILE__}" }
+  let(:event) { LogStash::Event.new("message" => message, "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+  let(:event_count) { 1 + rand(2) }
+  let(:config) { "not implemented" }
+  let(:events) { event_count.times.map { event }.to_a }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
+  
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) { "#{es_url}/#{index}" }
+
+  let(:curl_opts) { nil }
+
+  let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
+  let(:es_admin_pass) { 'elastic' }
+
+  let(:initial_events) { [] }
+
+  let(:do_register) { true }
+
+  before do
+    subject.register if do_register
+    subject.multi_receive(initial_events) if initial_events
+  end
+
+  after do
+    subject.do_close
+  end
+
+  shared_examples "an indexer" do |secure|
+    before(:each) do
+      host_unreachable_error_class = LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError
+      allow(host_unreachable_error_class).to receive(:new).with(any_args).and_wrap_original do |m, original, url|
+        if original.message.include?("PKIX path building failed")
+          $stderr.puts "Client not connecting due to PKIX path building failure; " +
+                         "shutting plugin down to prevent infinite retries"
+          subject.close # premature shutdown to prevent infinite retry
+        end
+        m.call(original, url)
+      end
+    end
+
+    it "ships events" do
+      subject.multi_receive(events)
+
+      curl_and_get_json_response "#{es_url}/_refresh", method: :post
+
+      result = curl_and_get_json_response "#{index_url}/_count?q=*"
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
+
+      result = curl_and_get_json_response "#{index_url}/_search?q=*&size=1000"
+      result["hits"]["hits"].each do |doc|
+        expect(doc["_source"]["message"]).to eq(message)
+
+        if ESHelper.es_version_satisfies?("< 8")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
+        expect(doc["_index"]).to eq(index)
+      end
+    end
+    
+    it "sets the correct content-type header" do
+      expected_manticore_opts = {:headers => {"Content-Type" => "application/json"}, :body => anything}
+      if secure
+        expected_manticore_opts = {
+          :headers => {"Content-Type" => "application/json"}, 
+          :body => anything, 
+          :auth => {
+            :user => user,
+            :password => password,
+            :eager => true
+          }}
+      end
+      expect(subject.client.pool.adapter.client).to receive(:send).
+        with(anything, anything, expected_manticore_opts).at_least(:once).
+        and_call_original
+      subject.multi_receive(events)
+    end
+  end
+
+  shared_examples "PKIX path failure" do
+    let(:do_register) { false }
+    let(:host_unreachable_error_class) { LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError }
+
+    before(:each) do
+      limit_execution
+    end
+
+    let(:limit_execution) do
+      Thread.new { sleep 5; subject.close }
+    end
+
+    it 'fails to establish TLS' do
+      allow(host_unreachable_error_class).to receive(:new).with(any_args).and_call_original.at_least(:once)
+
+      subject.register
+      limit_execution.join
+
+      sleep 1
+
+      expect(host_unreachable_error_class).to have_received(:new).at_least(:once) do |original, url|
+        expect(original.message).to include("PKIX path building failed")
+      end
+    end
+  end
+
+  describe "an indexer with custom index_type", :integration => true do
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index
+      }
+    }
+    it_behaves_like("an indexer")
+  end
+
+  describe "an indexer with no type value set (default to doc)", :integration => true do
+    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index
+      }
+    }
+    it_behaves_like("an indexer")
+  end
+
+  describe "a secured indexer", :secure_integration => true do
+    let(:user) { "simpleuser" }
+    let(:password) { "abc123" }
+    let(:cacert) { "spec/fixtures/test_certs/ca.crt" }
+    let(:es_url) { "https://#{get_host_port}" }
+    let(:config) do
+      {
+        "hosts" => [ get_host_port ],
+        "user" => user,
+        "password" => password,
+        "ssl_enabled" => true,
+        "ssl_certificate_authorities" => cacert,
+        "index" => index
+      }
+    end 
+
+    let(:curl_opts) { "-u #{user}:#{password}" }
+
+    if ENV['ES_SSL_KEY_INVALID'] == 'true' # test_invalid.crt (configured in ES) has SAN: DNS:localhost
+      # javax.net.ssl.SSLPeerUnverifiedException: Host name 'elasticsearch' does not match the certificate subject ...
+
+      context "when no keystore nor ca cert set and verification is disabled" do
+        let(:config) do
+          super().tap { |config| config.delete('ssl_certificate_authorities') }.merge('ssl_verification_mode' => 'none')
+        end
+
+        include_examples("an indexer", true)
+      end
+
+      context "when keystore is set and verification is disabled" do
+        let(:config) do
+          super().merge(
+              'ssl_verification_mode' => 'none',
+              'ssl_keystore_path' => 'spec/fixtures/test_certs/test.p12',
+              'ssl_keystore_password' => '1234567890'
+          )
+        end
+
+        include_examples("an indexer", true)
+      end
+
+      context "when keystore has self-signed cert and verification is disabled" do
+        let(:config) do
+          super().tap { |config| config.delete('ssl_certificate_authorities') }.merge(
+              'ssl_verification_mode' => 'none',
+              'ssl_keystore_path' => 'spec/fixtures/test_certs/test_self_signed.p12',
+              'ssl_keystore_password' => '1234567890'
+          )
+        end
+
+        include_examples("an indexer", true)
+      end
+
+    else
+
+      let(:curl_opts) { "#{super()} --tlsv1.2 --tls-max 1.3 -u #{es_admin}:#{es_admin_pass}" } # due ES 8.x we need user/password
+
+      it_behaves_like("an indexer", true)
+
+      describe "with a password requiring escaping" do
+        let(:user) { "f@ncyuser" }
+        let(:password) { "ab%12#" }
+
+        include_examples("an indexer", true)
+      end
+
+      describe "with a user/password requiring escaping in the URL" do
+        let(:config) do
+          {
+              "hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
+              "ssl_enabled" => true,
+              "ssl_certificate_authorities" => "spec/fixtures/test_certs/test.crt",
+              "index" => index
+          }
+        end
+
+        include_examples("an indexer", true)
+      end
+
+      context "without providing `ssl_certificate_authorities`" do
+        let(:config) do
+          super().tap do |c|
+            c.delete("ssl_certificate_authorities")
+          end
+        end
+
+        it_behaves_like("PKIX path failure")
+      end
+
+      if Gem::Version.new(LOGSTASH_VERSION) >= Gem::Version.new("8.3.0")
+        context "with `ca_trusted_fingerprint` instead of `ssl_certificate_authorities`" do
+          let(:config) do
+            super().tap do |c|
+              c.delete("ssl_certificate_authorities")
+              c.update("ca_trusted_fingerprint" => ca_trusted_fingerprint)
+            end
+          end
+          let(:ca_trusted_fingerprint) { File.read("spec/fixtures/test_certs/test.der.sha256").chomp }
+
+
+          it_behaves_like("an indexer", true)
+
+          context 'with an invalid `ca_trusted_fingerprint`' do
+            let(:ca_trusted_fingerprint) { super().reverse }
+
+            it_behaves_like("PKIX path failure")
+          end
+        end
+      end
+
+      context 'with enforced TLSv1.3 protocol' do
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.3' ] }
+
+        it_behaves_like("an indexer", true)
+      end
+
+      context 'with enforced TLSv1.2 protocol (while ES only enabled TLSv1.3)' do
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.2' ] }
+
+        let(:initial_events) { nil }
+
+        it "does not ship events" do
+          curl_and_get_json_response index_url, method: :put # make sure index exists
+          Thread.start { subject.multi_receive(events) } # we'll be stuck in a retry loop
+          sleep 2.5
+
+          curl_and_get_json_response "#{es_url}/_refresh", method: :post
+
+          result = curl_and_get_json_response "#{index_url}/_count?q=*"
+          cur_count = result["count"]
+          expect(cur_count).to eq(0) # ES output keeps re-trying but ends up with a
+          # [Manticore::ClientProtocolException] Received fatal alert: protocol_version
+        end
+
+      end if ENV['ES_SSL_SUPPORTED_PROTOCOLS'] == 'TLSv1.3'
+
+    end
+
+  end
+end

data/spec/integration/outputs/index_version_spec.rb

@@ -0,0 +1,98 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+
+describe "Versioned indexing", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  let(:es) { get_client }
+
+  before :each do
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    es.indices.delete(:index => "*") rescue nil
+    es.indices.refresh
+  end
+
+  context "when index only" do
+    subject { LogStash::Outputs::ElasticSearch.new(settings) }
+
+    before do
+      subject.register
+    end
+
+    describe "unversioned output" do
+      let(:settings) do
+        {
+          "manage_template" => true,
+          "index" => "logstash-index",
+          "template_overwrite" => true,
+          "hosts" => get_host_port(),
+          "action" => "index",
+          "document_id" => "%{my_id}"
+        }
+      end
+
+      it "should default to ES version" do
+        subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        expect(r["_version"]).to eq(1)
+        expect(r["_source"]["message"]).to eq('foo')
+        subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        expect(r2["_version"]).to eq(2)
+        expect(r2["_source"]["message"]).to eq('foobar')
+      end  
+    end
+
+    describe "versioned output" do
+      let(:settings) do 
+        {
+          "manage_template" => true,
+          "index" => "logstash-index",
+          "template_overwrite" => true,
+          "hosts" => get_host_port(),
+          "action" => "index",
+          "document_id" => "%{my_id}",
+          "version" => "%{my_version}",
+          "version_type" => "external",
+        }
+      end
+
+      it "should respect the external version" do
+        id = "ev1"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+      end
+
+      it "should ignore non-monotonic external version updates" do
+        id = "ev2"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r2["_version"]).to eq(99)
+        expect(r2["_source"]["message"]).to eq('foo')
+      end
+
+      it "should commit monotonic external version updates" do
+        id = "ev3"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r2["_version"]).to eq(100)
+        expect(r2["_source"]["message"]).to eq('foo')
+      end
+    end
+  end
+end

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -0,0 +1,75 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Ingest pipeline execution behavior", :integration => true do
+  subject! do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "hosts" => "#{get_host_port()}",
+      "pipeline" => "apache-logs",
+      "data_stream" => 'false',
+      "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  let(:http_client) { Manticore::Client.new }
+  let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
+  let(:apache_logs_pipeline) { '
+  {
+    "description" : "Pipeline to parse Apache logs",
+    "processors" : [
+      {
+        "grok": {
+          "field": "message",
+          "patterns": ["%{COMBINEDAPACHELOG}"]
+        }
+      }
+    ]
+  }'
+  }
+
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
+
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
+
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+
+    # delete existing ingest pipeline
+    http_client.delete(ingest_url).call
+
+    # register pipeline
+    http_client.put(ingest_url, :body => apache_logs_pipeline, :headers => {"Content-Type" => "application/json" }).call
+
+    #TODO: Use esclient
+    #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
+
+    subject.register
+    subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
+    @es.indices.refresh
+
+    #Wait or fail until everything's indexed.
+    Stud::try(10.times) do
+      r = @es.search(index: 'logstash-*')
+      expect(r).to have_hits(1)
+      sleep(0.1)
+    end
+  end
+
+  it "indexes using the proper pipeline" do
+    results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
+    expect(results).to have_hits(1)
+    expect(results["hits"]["hits"][0]["_source"]["response"]).to eq("200")
+    expect(results["hits"]["hits"][0]["_source"]["bytes"]).to eq("182")
+    expect(results["hits"]["hits"][0]["_source"]["verb"]).to eq("GET")
+    expect(results["hits"]["hits"][0]["_source"]["request"]).to eq("/scripts/netcat-webserver")
+    expect(results["hits"]["hits"][0]["_source"]["auth"]).to eq("-")
+    expect(results["hits"]["hits"][0]["_source"]["ident"]).to eq("-")
+    expect(results["hits"]["hits"][0]["_source"]["clientip"]).to eq("183.60.215.50")
+    expect(results["hits"]["hits"][0]["_source"]["junkfieldaaaa"]).to eq(nil)
+  end
+end

data/spec/integration/outputs/metrics_spec.rb

@@ -0,0 +1,66 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "metrics", :integration => true do
+  subject! do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "manage_template" => false,
+      "hosts" => "#{get_host_port()}"
+    }
+    plugin = LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  let(:metric) { subject.metric }
+  let(:bulk_request_metrics) { subject.instance_variable_get(:@bulk_request_metrics) }
+  let(:document_level_metrics) { subject.instance_variable_get(:@document_level_metrics) }
+
+  before :each do
+    require "elasticsearch"
+
+    # Clean ES of data before we start.
+    @es = get_client
+    clean(@es)
+    subject.register
+  end
+
+  context "after a succesful bulk insert" do
+    let(:bulk) { [
+      LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+      LogStash::Event.new("somevalue" => 100),
+      LogStash::Event.new("somevalue" => 10),
+      LogStash::Event.new("somevalue" => 1),
+      LogStash::Event.new("country" => "us"),
+      LogStash::Event.new("country" => "at"),
+      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+    ]}
+
+    it "increases successful bulk request metric" do
+      expect(bulk_request_metrics).to receive(:increment).with(:successes).once
+      subject.multi_receive(bulk)
+    end
+
+    it "increases number of successful inserted documents" do
+      expect(document_level_metrics).to receive(:increment).with(:successes, bulk.size).once
+      subject.multi_receive(bulk)
+    end
+  end
+
+  context "after a bulk insert that generates errors" do
+    let(:bulk) { [
+      LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("message" => { "message" => "sample nested message here" }),
+    ]}
+    it "increases bulk request with error metric" do
+      expect(bulk_request_metrics).to receive(:increment).with(:with_errors).once
+      expect(bulk_request_metrics).to_not receive(:increment).with(:successes)
+      subject.multi_receive(bulk)
+    end
+
+    it "increases number of successful and non retryable documents" do
+      expect(document_level_metrics).to receive(:increment).with(:dlq_routed).once
+      expect(document_level_metrics).to receive(:increment).with(:successes).once
+      subject.multi_receive(bulk)
+    end
+  end
+end