logstash-output-opensearch 1.0.0-java

data/spec/opensearch_spec_helper.rb

@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative './spec_helper'
+
+require 'elasticsearch'
+
+require 'json'
+require 'cabin'
+
+module OpenSearchHelper
+  def get_host_port
+    if ENV["INTEGRATION"] == "true"
+      "integration:9200"
+    else
+      "localhost:9200"
+    end
+  end
+
+  def get_client
+    Elasticsearch::Client.new(:hosts => [get_host_port])
+  end
+
+  def doc_type
+    "_doc"
+  end
+
+  def todays_date
+    Time.now.strftime("%Y.%m.%d")
+  end
+
+  def field_properties_from_template(template_name, field)
+    template = get_template(@client, template_name)
+    mappings = get_template_mappings(template)
+    mappings["properties"][field]["properties"]
+  end
+
+  def routing_field_name
+    :routing
+  end
+
+  def self.version
+    RSpec.configuration.filter[:version]
+  end
+
+  RSpec::Matchers.define :have_hits do |expected|
+    match do |actual|
+      expected == actual['hits']['total']['value']
+    end
+  end
+
+  RSpec::Matchers.define :have_index_pattern do |expected|
+    match do |actual|
+      test_against = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
+      test_against.include?(expected)
+    end
+  end
+
+  def clean(client)
+    client.indices.delete_template(:name => "*")
+    client.indices.delete_index_template(:name => "logstash*") rescue nil
+    # This can fail if there are no indexes, ignore failure.
+    client.indices.delete(:index => "*") rescue nil
+  end
+
+  def set_cluster_settings(client, cluster_settings)
+    client.cluster.put_settings(body: cluster_settings)
+    get_cluster_settings(client)
+  end
+
+  def get_cluster_settings(client)
+    client.cluster.get_settings
+  end
+
+  def put_alias(client, the_alias, index)
+    body = {
+        "aliases" => {
+            index => {
+                "is_write_index"=>  true
+            }
+        }
+    }
+    client.put_alias({name: the_alias, body: body})
+  end
+
+  def max_docs_policy(max_docs)
+  {
+    "policy" => {
+      "phases"=> {
+        "hot" => {
+          "actions" => {
+            "rollover" => {
+              "max_docs" => max_docs
+            }
+          }
+        }
+      }
+    }
+  }
+  end
+
+  def max_age_policy(max_age)
+  {
+    "policy" => {
+      "phases"=> {
+        "hot" => {
+          "actions" => {
+            "rollover" => {
+              "max_age" => max_age
+            }
+          }
+        }
+      }
+    }
+  }
+  end
+
+  def get_template(client, name)
+    t = client.indices.get_template(name: name)
+    # TODO: use index template if version >= 7.8 & OpenSearch
+    t[name]
+  end
+
+  def get_template_settings(template)
+    template['template']
+  end
+
+  def get_template_mappings(template)
+    template['mappings']
+  end
+end
+
+RSpec.configure do |config|
+  config.include OpenSearchHelper
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require "logstash/devutils/rspec/spec_helper"
+
+require "logstash/outputs/opensearch"
+
+module LogStash::Outputs::OpenSearch::SpecHelper
+end
+
+RSpec.configure do |config|
+  config.include LogStash::Outputs::OpenSearch::SpecHelper
+end

data/spec/unit/http_client_builder_spec.rb

@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/opensearch"
+require "logstash/outputs/opensearch/http_client"
+require "logstash/outputs/opensearch/http_client_builder"
+
+describe LogStash::Outputs::OpenSearch::HttpClientBuilder do
+  describe "auth setup with url encodable passwords" do
+    let(:klass) { LogStash::Outputs::OpenSearch::HttpClientBuilder }
+    let(:user) { "foo@bar"}
+    let(:password) {"baz@blah" }
+    let(:password_secured) do
+      secured = double("password")
+      allow(secured).to receive(:value).and_return(password)
+      secured
+    end
+    let(:options) { {"user" => user, "password" => password} }
+    let(:logger) { mock("logger") }
+    let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
+
+    it "should return the user escaped" do
+      expect(auth_setup[:user]).to eql(CGI.escape(user))
+    end
+
+    it "should return the password escaped" do
+      expect(auth_setup[:password]).to eql(CGI.escape(password))
+    end
+  end
+
+  describe "customizing action paths" do
+    let(:hosts) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200") ] }
+    let(:options) { {"hosts" => hosts } }
+    let(:logger) { double("logger") }
+    before :each do
+      [:debug, :debug?, :info?, :info, :warn].each do |level|
+        allow(logger).to receive(level)
+      end
+    end
+
+    describe "healthcheck_path" do
+
+      context "when setting bulk_path" do
+        let(:bulk_path) { "/meh" }
+        let(:options) { super().merge("bulk_path" => bulk_path) }
+
+        context "when using path" do
+          let(:options) { super().merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq(bulk_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the bulk_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq(bulk_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting bulk_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super().merge("path" => path) }
+          it "sets bulk_path to path+_bulk" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq("#{path}/_bulk")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the bulk_path to _bulk" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq("/_bulk")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+    describe "healthcheck_path" do
+      context "when setting healthcheck_path" do
+        let(:healthcheck_path) { "/meh" }
+        let(:options) { super().merge("healthcheck_path" => healthcheck_path) }
+
+        context "when using path" do
+          let(:options) { super().merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(healthcheck_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the healthcheck_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(healthcheck_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting healthcheck_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super().merge("path" => path) }
+          it "sets healthcheck_path to path" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the healthcheck_path to root" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq("/")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+    describe "sniffing_path" do
+      context "when setting sniffing_path" do
+        let(:sniffing_path) { "/meh" }
+        let(:options) { super().merge("sniffing_path" => sniffing_path) }
+
+        context "when using path" do
+          let(:options) { super().merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq(sniffing_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the sniffing_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq(sniffing_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting sniffing_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super().merge("path" => path) }
+          it "sets sniffing_path to path+_nodes/http" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq("#{path}/_nodes/http")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the sniffing_path to _nodes/http" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq("/_nodes/http")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require "logstash/outputs/opensearch"
+require_relative "../../../spec/opensearch_spec_helper"
+
+describe "whitelisting error types in expected behavior" do
+  let(:template) { '{"template" : "not important, will be updated by :index"}' }
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+  let(:action1) { ["index", {:_id=>1, :routing=>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1] }
+  let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } }
+
+  subject { LogStash::Outputs::OpenSearch.new(settings) }
+
+  before :each do
+    allow(subject.logger).to receive(:warn)
+    allow(subject).to receive(:maximum_seen_major_version).and_return(0)
+    allow(subject).to receive(:finish_register)
+
+    subject.register
+    allow(subject.client).to receive(:bulk).and_return(
+      {
+        "errors" => true,
+        "items" => [{
+          "create" => {
+            "status" => 409, 
+            "error" => {
+              "type" => "document_already_exists_exception",
+              "reason" => "[shard] document already exists"
+            }
+          }
+        }]
+      })
+
+    subject.multi_receive([event1])
+  end
+
+  after :each do
+    subject.close
+  end
+
+  describe "when failure logging is enabled for everything" do
+    it "should log a failure on the action" do
+      expect(subject.logger).to have_received(:warn).with("Failed action", anything)
+    end
+  end
+
+  describe "when failure logging is disabled for docuemnt exists error" do
+    let(:settings) { super().merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
+
+    it "should log a failure on the action" do
+      expect(subject.logger).not_to have_received(:warn).with("Failed action", anything)
+    end
+  end
+
+end

data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb

@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/opensearch/http_client"
+require 'cabin'
+
+describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
+  let(:logger) { Cabin::Channel.get }
+  let(:options) { {} }
+
+  subject { described_class.new(logger, options) }
+
+  it "should raise an exception if requests are issued after close" do
+    subject.close
+    expect { subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
+  end
+
+  it "should implement host unreachable exceptions" do
+    expect(subject.host_unreachable_exceptions).to be_a(Array)
+  end
+  
+  describe "auth" do
+    let(:user) { "myuser" }
+    let(:password) { "mypassword" }
+    let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
+    
+    it "should convert the auth to params" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(200)
+      
+      expected_uri = noauth_uri.clone
+      expected_uri.path = "/"
+      
+      expect(subject.manticore).to receive(:get).
+        with(expected_uri.to_s, {
+          :headers => {"Content-Type" => "application/json"},
+          :auth => {
+            :user => user,
+            :password => password,
+            :eager => true
+          }
+        }).and_return resp
+      
+      subject.perform_request(uri, :get, "/")
+    end
+  end
+
+  describe "bad response codes" do
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+
+    it "should raise a bad response code error" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(500)
+      allow(resp).to receive(:body).and_return("a body")
+
+      expect(subject.manticore).to receive(:get).
+        with(uri.to_s + "/", anything).
+        and_return(resp)
+
+      uri_with_path = uri.clone
+      uri_with_path.path = "/"
+
+      expect(::LogStash::Outputs::OpenSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
+        with(resp.code, uri_with_path, nil, resp.body).and_call_original
+
+      expect do
+        subject.perform_request(uri, :get, "/")
+      end.to raise_error(::LogStash::Outputs::OpenSearch::HttpClient::Pool::BadResponseCodeError)
+    end
+  end
+
+  describe "format_url" do
+    let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
+    let(:path) { "_bulk" }
+    subject { described_class.new(double("logger"), {}) }
+
+    it "should add the path argument to the uri's path" do
+      expect(subject.format_url(url, path).path).to eq("/path/_bulk")
+    end
+
+    context "when uri contains query parameters" do
+      let(:query_params) { "query=value&key=value2" }
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
+      let(:formatted) { subject.format_url(url, path)}
+
+      it "should retain query_params after format" do
+        expect(formatted.query).to eq(query_params)
+      end
+      
+      context "and the path contains query parameters" do
+        let(:path) { "/special_path?specialParam=123" }
+        
+        it "should join the query correctly" do
+          expect(formatted.query).to eq(query_params + "&specialParam=123")
+        end
+      end
+    end
+    
+    context "when the path contains query parameters" do
+      let(:path) { "/special_bulk?pathParam=1"}
+      let(:formatted) { subject.format_url(url, path) }
+      
+      it "should add the path correctly" do
+        expect(formatted.path).to eq("#{url.path}special_bulk")
+        expect(subject.remove_double_escaping(formatted.path)).to eq("#{url.path}special_bulk")
+      end 
+      
+      it "should add the query parameters correctly" do
+        expect(formatted.query).to eq("pathParam=1")
+      end
+    end
+
+    context "when uri contains credentials" do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it "should remove credentials after format" do
+        expect(formatted.userinfo).to be_nil
+      end
+    end
+
+    context 'when uri contains date math' do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+      let(:path) { CGI.escape("<logstash-{now/d}-0001>") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it 'should escape the uri correctly' do
+        expect(subject.remove_double_escaping(formatted.path)).to eq("/%3Clogstash-%7Bnow%2Fd%7D-0001%3E")
+      end
+    end
+
+    context 'when uri does not contain date math' do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+      let(:path) { CGI.escape("logstash-0001") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it 'should escape the uri correctly' do
+        expect(subject.remove_double_escaping(formatted.path)).to eq("/logstash-0001")
+      end
+    end
+  end
+
+  describe "integration specs", :integration => true do
+    it "should perform correct tests without error" do
+      resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")
+      expect(resp.code).to eql(200)
+    end
+  end
+end