logstash-output-opensearch 1.0.0-java

data/spec/fixtures/scripts/painless/scripted_update.painless

@@ -0,0 +1,2 @@
+ctx._source.counter += params.event.count
+

data/spec/fixtures/scripts/painless/scripted_update_nested.painless

@@ -0,0 +1 @@
+ctx._source.counter += params.event.data.count

data/spec/fixtures/scripts/painless/scripted_upsert.painless

@@ -0,0 +1 @@
+ctx._source.counter = params.event.counter

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+require "logstash/outputs/opensearch"
+require "stringio"
+
+RSpec::Matchers.define :a_valid_gzip_encoded_string do
+  match { |data|
+    expect { Zlib::GzipReader.new(StringIO.new(data)).read }.not_to raise_error
+  }
+end
+
+
+describe "indexing with http_compression turned on", :integration => true do
+  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { "_doc" }
+  let(:event_count) { 10000 + rand(500) }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+    {
+      "hosts" => get_host_port,
+      "index" => index,
+      "http_compression" => true
+    }
+  }
+  subject { LogStash::Outputs::OpenSearch.new(config) }
+
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{es_url}/#{index}"}
+  let(:http_client_options) { {} }
+  let(:http_client) do
+    Manticore::Client.new(http_client_options)
+  end
+
+  before do
+    subject.register
+    subject.multi_receive([])
+  end
+
+  shared_examples "an indexer" do
+    it "ships events" do
+      subject.multi_receive(events)
+
+      http_client.post("#{es_url}/_refresh").call
+
+      response = http_client.get("#{index_url}/_count?q=*")
+      result = LogStash::Json.load(response.body)
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
+
+      response = http_client.get("#{index_url}/_search?q=*&size=1000")
+      result = LogStash::Json.load(response.body)
+      result["hits"]["hits"].each do |doc|
+        expect(doc["_type"]).to eq(type)
+        expect(doc["_index"]).to eq(index)
+      end
+    end
+  end
+
+  it "sets the correct content-encoding header and body is compressed" do
+    expect(subject.client.pool.adapter.client).to receive(:send).
+      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+      and_call_original
+    subject.multi_receive(events)
+  end
+
+  it_behaves_like("an indexer")
+end

data/spec/integration/outputs/create_spec.rb

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+
+describe "client create actions", :integration => true do
+  require "logstash/outputs/opensearch"
+
+  def get_es_output(action, id, version=nil, version_type=nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-create",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => action
+    }
+    settings['document_id'] = id
+    settings['version'] = version if version
+    settings['version_type'] = version_type if version_type
+    LogStash::Outputs::OpenSearch.new(settings)
+  end
+
+  before :each do
+    @client = get_client
+    # Delete all templates first.
+    # Clean OpenSearch of data before we start.
+    @client.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @client.indices.delete(:index => "*") rescue nil
+  end
+
+  context "when action => create" do
+    it "should create new documents with or without id" do
+      subject = get_es_output("create", "id123")
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      @client.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @client.search(index: 'logstash-*')
+        expect(r).to have_hits(1)
+      end
+    end
+
+    it "should allow default (internal) version" do
+      subject = get_es_output("create", "id123", 43)
+      subject.register
+    end
+
+    it "should allow internal version" do
+      subject = get_es_output("create", "id123", 43, "internal")
+      subject.register
+    end
+
+    it "should not allow external version" do
+      subject = get_es_output("create", "id123", 43, "external")
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should not allow external_gt version" do
+      subject = get_es_output("create", "id123", 43, "external_gt")
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should not allow external_gte version" do
+      subject = get_es_output("create", "id123", 43, "external_gte")
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+end

data/spec/integration/outputs/delete_spec.rb

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+require "logstash/outputs/opensearch"
+
+
+describe "Versioned delete", :integration => true do
+  require "logstash/outputs/opensearch"
+
+  let(:es) { get_client }
+
+  before :each do
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    es.indices.delete(:index => "*") rescue nil
+    es.indices.refresh
+  end
+
+  context "when delete only" do
+    subject { LogStash::Outputs::OpenSearch.new(settings) }
+
+    before do
+      subject.register
+    end
+
+    let(:settings) do
+      {
+        "manage_template" => true,
+        "index" => "logstash-delete",
+        "template_overwrite" => true,
+        "hosts" => get_host_port(),
+        "document_id" => "%{my_id}",
+        "version" => "%{my_version}",
+        "version_type" => "external",
+        "action" => "%{my_action}"
+      }
+    end
+
+    it "should ignore non-monotonic external version updates" do
+      id = "ev2"
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r['_version']).to eq(99)
+      expect(r['_source']['message']).to eq('foo')
+
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
+      r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r2['_version']).to eq(99)
+      expect(r2['_source']['message']).to eq('foo')
+    end
+
+    it "should commit monotonic external version updates" do
+      id = "ev3"
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r['_version']).to eq(99)
+      expect(r['_source']['message']).to eq('foo')
+
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
+      expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+    end
+  end
+end

data/spec/integration/outputs/index_spec.rb

@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+require "logstash/outputs/opensearch"
+
+describe "TARGET_BULK_BYTES", :integration => true do
+  let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+  let(:event_count) { 1000 }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index
+      }
+  }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { "_doc" }
+
+  subject { LogStash::Outputs::OpenSearch.new(config) }
+
+  before do
+    subject.register
+    allow(subject.client).to receive(:bulk_send).with(any_args).and_call_original
+    subject.multi_receive(events)
+  end
+
+  describe "batches that are too large for one" do
+    let(:event) { LogStash::Event.new("message" => "a " * (((target_bulk_bytes/2) / event_count)+1)) }
+
+    it "should send in two batches" do
+      expect(subject.client).to have_received(:bulk_send).twice do |payload|
+        expect(payload.size).to be <= target_bulk_bytes
+      end
+    end
+
+    describe "batches that fit in one" do
+      # Normally you'd want to generate a request that's just 1 byte below the limit, but it's
+      # impossible to know how many bytes an event will serialize as with bulk proto overhead
+      let(:event) { LogStash::Event.new("message" => "a") }
+
+      it "should send in one batch" do
+        expect(subject.client).to have_received(:bulk_send).once do |payload|
+          expect(payload.size).to be <= target_bulk_bytes
+        end
+      end
+    end
+  end
+end
+
+describe "indexing" do
+  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { "_doc" }
+  let(:event_count) { 1 + rand(2) }
+  let(:config) { "not implemented" }
+  let(:events) { event_count.times.map { event }.to_a }
+  subject { LogStash::Outputs::OpenSearch.new(config) }
+  
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{es_url}/#{index}"}
+  let(:http_client_options) { {} }
+  let(:http_client) do
+    Manticore::Client.new(http_client_options)
+  end
+
+  before do
+    subject.register
+    subject.multi_receive([])
+  end
+  
+  shared_examples "an indexer" do |secure|
+    it "ships events" do
+      subject.multi_receive(events)
+
+      http_client.post("#{es_url}/_refresh").call
+
+      response = http_client.get("#{index_url}/_count?q=*")
+      result = LogStash::Json.load(response.body)
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
+
+      response = http_client.get("#{index_url}/_search?q=*&size=1000")
+      result = LogStash::Json.load(response.body)
+      result["hits"]["hits"].each do |doc|
+        expect(doc["_type"]).to eq(type)
+        expect(doc["_index"]).to eq(index)
+      end
+    end
+    
+    it "sets the correct content-type header" do
+      expected_manticore_opts = {:headers => {"Content-Type" => "application/json"}, :body => anything}
+      if secure
+        expected_manticore_opts = {
+          :headers => {"Content-Type" => "application/json"}, 
+          :body => anything, 
+          :auth => {
+            :user => user,
+            :password => password,
+            :eager => true
+          }}
+      end
+      expect(subject.client.pool.adapter.client).to receive(:send).
+        with(anything, anything, expected_manticore_opts).at_least(:once).
+        and_call_original
+      subject.multi_receive(events)
+    end
+  end
+
+  describe "an indexer with custom index_type", :integration => true do
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index
+      }
+    }
+    it_behaves_like("an indexer")
+  end
+
+  describe "an indexer with no type value set (default to doc)", :integration => true do
+    let(:type) { "_doc" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index
+      }
+    }
+    it_behaves_like("an indexer")
+  end
+  describe "a secured indexer", :secure_integration => true do
+    let(:user) { "admin" }
+    let(:password) { "admin" }
+    let(:es_url) {"https://integration:9200"}
+    let(:config) do
+      {
+        "hosts" => ["integration:9200"],
+        "user" => user,
+        "password" => password,
+        "ssl" => true,
+        "ssl_certificate_verification" => false,
+        "index" => index
+      }
+    end
+    let(:http_client_options) do
+      {
+        :auth => {
+          :user => user,
+          :password => password
+        },
+        :ssl => {
+          :enabled => true,
+          :verify => false
+        }
+      }
+    end
+    it_behaves_like("an indexer", true)
+  end
+end

data/spec/integration/outputs/index_version_spec.rb

@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+require "logstash/outputs/opensearch"
+
+describe "Versioned indexing", :integration => true do
+  require "logstash/outputs/opensearch"
+
+  let(:es) { get_client }
+
+  before :each do
+    # Delete all templates first.
+    # Clean OpenSearch of data before we start.
+    es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    es.indices.delete(:index => "*") rescue nil
+    es.indices.refresh
+  end
+
+  context "when index only" do
+    subject { LogStash::Outputs::OpenSearch.new(settings) }
+
+    before do
+      subject.register
+    end
+
+    describe "unversioned output" do
+      let(:settings) do
+        {
+          "manage_template" => true,
+          "index" => "logstash-index",
+          "template_overwrite" => true,
+          "hosts" => get_host_port(),
+          "action" => "index",
+          "script_lang" => "groovy",
+          "document_id" => "%{my_id}"
+        }
+      end
+
+      it "should default to OpenSearch version" do
+        subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        expect(r["_version"]).to eq(1)
+        expect(r["_source"]["message"]).to eq('foo')
+        subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        expect(r2["_version"]).to eq(2)
+        expect(r2["_source"]["message"]).to eq('foobar')
+      end
+    end
+
+    describe "versioned output" do
+      let(:settings) do
+        {
+          "manage_template" => true,
+          "index" => "logstash-index",
+          "template_overwrite" => true,
+          "hosts" => get_host_port(),
+          "action" => "index",
+          "script_lang" => "groovy",
+          "document_id" => "%{my_id}",
+          "version" => "%{my_version}",
+          "version_type" => "external",
+        }
+      end
+
+      it "should respect the external version" do
+        id = "ev1"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+      end
+
+      it "should ignore non-monotonic external version updates" do
+        id = "ev2"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r2["_version"]).to eq(99)
+        expect(r2["_source"]["message"]).to eq('foo')
+      end
+
+      it "should commit monotonic external version updates" do
+        id = "ev3"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        expect(r2["_version"]).to eq(100)
+        expect(r2["_source"]["message"]).to eq('foo')
+      end
+    end
+  end
+end
+