logstash-output-opensearch 1.0.0-java

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+
+describe "Ingest pipeline execution behavior", :integration => true do
+  subject! do
+    require "logstash/outputs/opensearch"
+    settings = {
+      "hosts" => "#{get_host_port()}",
+      "pipeline" => "apache-logs"
+    }
+    next LogStash::Outputs::OpenSearch.new(settings)
+  end
+
+  let(:http_client) { Manticore::Client.new }
+  let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
+  let(:apache_logs_pipeline) { '
+  {
+    "description" : "Pipeline to parse Apache logs",
+    "processors" : [
+      {
+        "grok": {
+          "field": "message",
+          "patterns": ["%{COMBINEDAPACHELOG}"]
+        }
+      }
+    ]
+  }'
+  }
+
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
+
+    # Clean OpenSearch of data before we start.
+    @client = get_client
+    @client.indices.delete_template(:name => "*")
+
+    # This can fail if there are no indexes, ignore failure.
+    @client.indices.delete(:index => "*") rescue nil
+
+    # delete existing ingest pipeline
+    http_client.delete(ingest_url).call
+
+    # register pipeline
+    http_client.put(ingest_url, :body => apache_logs_pipeline, :headers => {"Content-Type" => "application/json" }).call
+
+    #TODO: Use esclient
+    #@client.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
+
+    subject.register
+    subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
+    @client.indices.refresh
+
+    #Wait or fail until everything's indexed.
+    Stud::try(10.times) do
+      r = @client.search(index: 'logstash-*')
+      expect(r).to have_hits(1)
+      sleep(0.1)
+    end
+  end
+
+  it "indexes using the proper pipeline" do
+    results = @client.search(:index => 'logstash-*', :q => "message:\"netcat\"")
+    expect(results).to have_hits(1)
+    expect(results["hits"]["hits"][0]["_source"]["response"]).to eq("200")
+    expect(results["hits"]["hits"][0]["_source"]["bytes"]).to eq("182")
+    expect(results["hits"]["hits"][0]["_source"]["verb"]).to eq("GET")
+    expect(results["hits"]["hits"][0]["_source"]["request"]).to eq("/scripts/netcat-webserver")
+    expect(results["hits"]["hits"][0]["_source"]["auth"]).to eq("-")
+    expect(results["hits"]["hits"][0]["_source"]["ident"]).to eq("-")
+    expect(results["hits"]["hits"][0]["_source"]["clientip"]).to eq("183.60.215.50")
+    expect(results["hits"]["hits"][0]["_source"]["junkfieldaaaa"]).to eq(nil)
+  end
+end

data/spec/integration/outputs/metrics_spec.rb

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+
+describe "metrics", :integration => true do
+  subject! do
+    require "logstash/outputs/opensearch"
+    settings = {
+      "manage_template" => false,
+      "hosts" => "#{get_host_port()}"
+    }
+    plugin = LogStash::Outputs::OpenSearch.new(settings)
+  end
+
+  let(:metric) { subject.metric }
+  let(:bulk_request_metrics) { subject.instance_variable_get(:@bulk_request_metrics) }
+  let(:document_level_metrics) { subject.instance_variable_get(:@document_level_metrics) }
+
+  before :each do
+    require "elasticsearch"
+
+    # Clean OpenSearch of data before we start.
+    @client = get_client
+    clean(@client)
+    subject.register
+  end
+
+  context "after a succesful bulk insert" do
+    let(:bulk) { [
+      LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+      LogStash::Event.new("somevalue" => 100),
+      LogStash::Event.new("somevalue" => 10),
+      LogStash::Event.new("somevalue" => 1),
+      LogStash::Event.new("country" => "us"),
+      LogStash::Event.new("country" => "at"),
+      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+    ]}
+
+    it "increases successful bulk request metric" do
+      expect(bulk_request_metrics).to receive(:increment).with(:successes).once
+      subject.multi_receive(bulk)
+    end
+
+    it "increases number of successful inserted documents" do
+      expect(document_level_metrics).to receive(:increment).with(:successes, bulk.size).once
+      subject.multi_receive(bulk)
+    end
+  end
+
+  context "after a bulk insert that generates errors" do
+    let(:bulk) { [
+      LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("message" => { "message" => "sample nested message here" }),
+    ]}
+    it "increases bulk request with error metric" do
+      expect(bulk_request_metrics).to receive(:increment).with(:with_errors).once
+      expect(bulk_request_metrics).to_not receive(:increment).with(:successes)
+      subject.multi_receive(bulk)
+    end
+
+    it "increases number of successful and non retryable documents" do
+      expect(document_level_metrics).to receive(:increment).with(:non_retryable_failures).once
+      expect(document_level_metrics).to receive(:increment).with(:successes).once
+      subject.multi_receive(bulk)
+    end
+  end
+end

data/spec/integration/outputs/no_opensearch_on_startup_spec.rb

@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require "logstash/outputs/opensearch"
+require_relative "../../../spec/opensearch_spec_helper"
+
+describe "opensearch is down on startup", :integration => true do
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:event2) { LogStash::Event.new("message" => "a") }
+
+  subject {
+    LogStash::Outputs::OpenSearch.new({
+                                           "manage_template" => true,
+                                           "index" => "logstash-2014.11.17",
+                                           "template_overwrite" => true,
+                                           "hosts" => get_host_port(),
+                                           "retry_max_interval" => 64,
+                                           "retry_initial_interval" => 2
+                                       })
+  }
+
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
+    allow(Stud).to receive(:stoppable_sleep)
+
+    # Clean OpenSearch of data before we start.
+    @client = get_client
+    @client.indices.delete_template(:name => "*")
+    @client.indices.delete(:index => "*")
+    @client.indices.refresh
+  end
+
+  after :each do
+    subject.close
+  end
+
+  it 'should ingest events when OpenSearch recovers before documents are sent' do
+    allow_any_instance_of(LogStash::Outputs::OpenSearch::HttpClient::Pool).to receive(:get_version).and_raise(::LogStash::Outputs::OpenSearch::HttpClient::Pool::HostUnreachableError.new(StandardError.new, "big fail"))
+    subject.register
+    allow_any_instance_of(LogStash::Outputs::OpenSearch::HttpClient::Pool).to receive(:get_version).and_return(OpenSearchHelper.version)
+    subject.multi_receive([event1, event2])
+    @client.indices.refresh
+    r = @client.search(index: 'logstash-*')
+    expect(r).to have_hits(2)
+  end
+
+  it 'should ingest events when OpenSearch recovers after documents are sent' do
+    allow_any_instance_of(LogStash::Outputs::OpenSearch::HttpClient::Pool).to receive(:get_version).and_raise(::LogStash::Outputs::OpenSearch::HttpClient::Pool::HostUnreachableError.new(StandardError.new, "big fail"))
+    subject.register
+    Thread.new do
+      sleep 4
+      allow_any_instance_of(LogStash::Outputs::OpenSearch::HttpClient::Pool).to receive(:get_version).and_return(OpenSearchHelper.version)
+    end
+    subject.multi_receive([event1, event2])
+    @client.indices.refresh
+    r = @client.search(index: 'logstash-*')
+    expect(r).to have_hits(2)
+  end
+
+end

data/spec/integration/outputs/painless_update_spec.rb

@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+
+describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
+  require "logstash/outputs/opensearch"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "update"
+    }
+    LogStash::Outputs::OpenSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @client = get_client
+    # Delete all templates first.
+    # Clean OpenSearch of data before we start.
+    @client.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @client.indices.delete(:index => "*") rescue nil
+    @client.index(
+      :index => 'logstash-update',
+      :type => doc_type,
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
+    @client.indices.refresh
+  end
+
+  context "scripted updates" do
+
+    it "should increment a counter with event/doc 'count' variable with inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'ctx._source.counter += params.event.counter',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+      expect(r["_source"]["counter"]).to eq(4)
+    end
+
+    it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'doc_as_upsert' => true,
+        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+      expect(r["_source"]["counter"]).to eq(4)
+    end
+
+    it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+      subject = get_es_output({
+        'document_id' => "456",
+        'doc_as_upsert' => true,
+        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["counter"]).to eq(3)
+    end
+
+    context 'with an indexed script' do
+      it "should increment a counter with event/doc 'count' variable with indexed script" do
+        @client.perform_request(:put, "_scripts/indexed_update", {}, {"script" => {"source" => "ctx._source.counter += params.event.count", "lang" => "painless"}})
+        plugin_parameters = {
+          'document_id' => "123",
+          'script' => 'indexed_update',
+          'script_type' => 'indexed'
+        }
+
+        plugin_parameters.merge!('script_lang' => '')
+
+        subject = get_es_output(plugin_parameters)
+        subject.register
+        subject.multi_receive([LogStash::Event.new("count" => 4 )])
+        r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+        expect(r["_source"]["counter"]).to eq(5)
+      end
+    end
+   end
+
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["message"]).to eq('upsert message')
+    end
+
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["message"]).to eq('sample message here')
+    end
+
+    it "should fail on documents with event/doc as upsert at external version" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "updates with scripted upsert" do
+
+    context 'with an inline script' do
+      it "should create new documents with upsert content" do
+        subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
+        subject.register
+
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+        expect(r["_source"]["message"]).to eq('upsert message')
+      end
+
+      it "should create new documents with event/doc as script params" do
+        subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("counter" => 1)])
+        @client.indices.refresh
+        r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+        expect(r["_source"]["counter"]).to eq(1)
+      end
+    end
+  end
+end

data/spec/integration/outputs/parent_spec.rb

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../spec/opensearch_spec_helper"
+require "logstash/outputs/opensearch"
+
+context "join field tests", :integration => true do
+
+  shared_examples "a join field based parent indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+
+    let(:type) { "_doc" }
+
+    let(:event_count) { 10000 + rand(500) }
+    let(:parent) { "not_implemented" }
+    let(:config) { "not_implemented" }
+    let(:parent_id) { "test" }
+    let(:join_field) { "join_field" }
+    let(:parent_relation) { "parent_type" }
+    let(:child_relation) { "child_type" }
+    let(:default_headers) {
+      {"Content-Type" => "application/json"}
+    }
+    subject { LogStash::Outputs::OpenSearch.new(config) }
+
+    before do
+      # Add mapping and a parent document
+      index_url = "http://#{get_host_port()}/#{index}"
+
+      properties = {
+        "properties" => {
+          join_field => {
+            "type" => "join",
+            "relations" => { parent_relation => child_relation }
+          }
+        }
+      }
+
+      mapping = { "mappings" => properties}
+
+      Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
+      pdoc = { "message" => "ohayo", join_field => parent_relation }
+      Manticore.put("#{index_url}/#{type}/#{parent_id}", {:body => pdoc.to_json, :headers => default_headers}).call
+
+      subject.register
+      subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => parent_id, "message" => "Hello World!", join_field => child_relation) })
+    end
+
+
+    it "ships events" do
+      index_url = "http://#{get_host_port()}/#{index}"
+
+      Manticore.post("#{index_url}/_refresh").call
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        query = { "query" => { "has_parent" => { "parent_type" => parent_relation, "query" => { "match_all" => { } } } } }
+        response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
+        data = response.body
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        expect(cur_count).to eq(event_count)
+      end
+    end
+  end
+
+  describe "(http protocol) index events with static parent" do
+    it_behaves_like 'a join field based parent indexer' do
+      let(:config) {
+        {
+          "hosts" => get_host_port,
+          "index" => index,
+          "parent" => parent_id,
+          "document_type" => type,
+          "join_field" => join_field,
+          "manage_template" => false
+        }
+      }
+    end
+  end
+
+  describe "(http_protocol) index events with fieldref in parent value" do
+    it_behaves_like 'a join field based parent indexer' do
+      let(:config) {
+        {
+          "hosts" => get_host_port,
+          "index" => index,
+          "parent" => "%{link_to}",
+          "document_type" => type,
+          "join_field" => join_field,
+          "manage_template" => false
+        }
+      }
+    end
+  end
+end
+