logstash-output-opensearch 1.2.0-java → 1.3.0-java

data/lib/logstash/outputs/opensearch.rb

@@ -78,7 +78,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   include(LogStash::PluginMixins::OpenSearch::Common)
 
   # ecs_compatibility option, provided by Logstash core or the support adapter.
-  include(LogStash::PluginMixins::ECSCompatibilitySupport)
+  include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
 
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::OpenSearch::APIConfigs)
@@ -419,7 +419,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
     when :disabled
       @default_index = "logstash-%{+yyyy.MM.dd}"
       @default_template_name = 'logstash'
-    when :v1
+    when :v1, :v8
       @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
       @default_template_name = 'ecs-logstash'
     else

data/logstash-output-opensearch.gemspec

@@ -11,7 +11,7 @@ signing_key_path = "gem-private_key.pem"
 
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-opensearch'
-  s.version         = '1.2.0'
+  s.version         = '1.3.0'
 
   s.licenses        = ['Apache-2.0']
   s.summary         = "Stores logs in OpenSearch"
@@ -46,10 +46,11 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
   s.add_runtime_dependency 'aws-sdk',  '>= 2.11.632', '~> 2'
+  s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
   s.add_development_dependency 'cabin', ['~> 0.6']
-  s.add_development_dependency 'opensearch-ruby'
+  s.add_development_dependency 'opensearch-ruby', '~> 1'
 end

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -59,7 +59,13 @@ describe "indexing with http_compression turned on", :integration => true do
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x
+        # need a cleaner way to check this.
+        if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
         expect(doc["_index"]).to eq(index)
       end
     end

data/spec/integration/outputs/index_spec.rb

@@ -88,7 +88,13 @@ describe "indexing" do
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x
+        # need a cleaner way to check this.
+        if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
         expect(doc["_index"]).to eq(index)
       end
     end

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -14,7 +14,8 @@ describe "Ingest pipeline execution behavior", :integration => true do
     require "logstash/outputs/opensearch"
     settings = {
       "hosts" => "#{get_host_port()}",
-      "pipeline" => "apache-logs"
+      "pipeline" => "apache-logs",
+      "ecs_compatibility" => "disabled" # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::OpenSearch.new(settings)
   end

data/spec/integration/outputs/parent_spec.rb

@@ -10,7 +10,7 @@
 require_relative "../../../spec/opensearch_spec_helper"
 require "logstash/outputs/opensearch"
 
-context "join field tests", :integration => true do
+context "join field tests", :integration => true && OpenSearchHelper.check_version?("<2") do
 
   shared_examples "a join field based parent indexer" do
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }

data/spec/integration/outputs/retry_spec.rb

@@ -46,7 +46,8 @@ describe "failures in bulk class expected behavior", :integration => true do
       "template_overwrite" => true,
       "hosts" => get_host_port(),
       "retry_max_interval" => 64,
-      "retry_initial_interval" => 2
+      "retry_initial_interval" => 2,
+      "ecs_compatibility" => "disabled" # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::OpenSearch.new(settings)
   end

data/spec/integration/outputs/templates_spec.rb

@@ -10,8 +10,12 @@
 require_relative "../../../spec/opensearch_spec_helper"
 
 describe "index template expected behavior", :integration => true do
+  let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
+
   subject! do
     require "logstash/outputs/opensearch"
+    allow_any_instance_of(LogStash::Outputs::OpenSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+
     settings = {
       "manage_template" => true,
       "template_overwrite" => true,
@@ -29,75 +33,97 @@ describe "index template expected behavior", :integration => true do
     # This can fail if there are no indexes, ignore failure.
     @client.indices.delete(:index => "*") rescue nil
 
-    subject.register
-
-    subject.multi_receive([
-      LogStash::Event.new("message" => "sample message here"),
-      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-      LogStash::Event.new("somevalue" => 100),
-      LogStash::Event.new("somevalue" => 10),
-      LogStash::Event.new("somevalue" => 1),
-      LogStash::Event.new("country" => "us"),
-      LogStash::Event.new("country" => "at"),
-      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-    ])
-
-    @client.indices.refresh
-
-    # Wait or fail until everything's indexed.
-    Stud::try(20.times) do
-      r = @client.search(index: 'logstash-*')
-      expect(r).to have_hits(8)
-    end
   end
 
-  it "permits phrase searching on string fields" do
-    results = @client.search(:q => "message:\"sample message\"")
-    expect(results).to have_hits(1)
-    expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
-  end
+  context 'with ecs_compatibility => disabled' do
+    let(:ecs_compatibility) { :disabled }
+
+    before :each do
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @client.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @client.search(index: 'logstash-*')
+        expect(r).to have_hits(8)
+      end
+    end
 
-  it "numbers dynamically map to a numeric type and permit range queries" do
-    results = @client.search(:q => "somevalue:[5 TO 105]")
-    expect(results).to have_hits(2)
+    it "permits phrase searching on string fields" do
+      results = @client.search(:q => "message:\"sample message\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
+    end
 
-    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-    expect(values).to include(10)
-    expect(values).to include(100)
-    expect(values).to_not include(1)
-  end
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @client.search(:q => "somevalue:[5 TO 105]")
+      expect(results).to have_hits(2)
 
-  it "does not create .keyword field for top-level message field" do
-    results = @client.search(:q => "message.keyword:\"sample message here\"")
-    expect(results).to have_hits(0)
-  end
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      expect(values).to include(10)
+      expect(values).to include(100)
+      expect(values).to_not include(1)
+    end
 
-  it "creates .keyword field for nested message fields" do
-    results = @client.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
-    expect(results).to have_hits(1)
-  end
+    it "does not create .keyword field for top-level message field" do
+      results = @client.search(:q => "message.keyword:\"sample message here\"")
+      expect(results).to have_hits(0)
+    end
 
-  it "creates .keyword field from any string field which is not_analyzed" do
-    results = @client.search(:q => "country.keyword:\"us\"")
-    expect(results).to have_hits(1)
-    expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+    it "creates .keyword field for nested message fields" do
+      results = @client.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+      expect(results).to have_hits(1)
+    end
 
-    # partial or terms should not work.
-    results = @client.search(:q => "country.keyword:\"u\"")
-    expect(results).to have_hits(0)
-  end
+    it "creates .keyword field from any string field which is not_analyzed" do
+      results = @client.search(:q => "country.keyword:\"us\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+
+      # partial or terms should not work.
+      results = @client.search(:q => "country.keyword:\"u\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "make [geoip][location] a geo_point" do
+      expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+    end
 
-  it "make [geoip][location] a geo_point" do
-    expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+    it "aggregate .keyword results correctly " do
+      results = @client.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
+
+      expect(terms).to include("us")
+
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      expect(terms).to include("at")
+    end
   end
 
-  it "aggregate .keyword results correctly " do
-    results = @client.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
-    terms = results["buckets"].collect { |b| b["key"] }
+  context 'with ECS enabled' do
+    let(:ecs_compatibility) { :v1 }
 
-    expect(terms).to include("us")
+    before(:each) do
+      subject.register # should load template?
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+    end
 
-    # 'at' is a stopword, make sure stopwords are not ignored.
-    expect(terms).to include("at")
+    it 'loads the templates' do
+      aggregate_failures do
+        expect(@client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
+      end
+    end
   end
 end

data/spec/opensearch_spec_helper.rb

@@ -49,6 +49,16 @@ module OpenSearchHelper
     RSpec.configuration.filter[:version]
   end
 
+  def self.check_version?(*requirement)
+    version = self.version
+    if version.nil? || version.empty?
+      puts "version tag isn't set. Returning false from 'check_version?`."
+      return false
+    end
+    release_version = Gem::Version.new(version).release
+    Gem::Requirement.new(requirement).satisfied_by?(release_version)
+  end
+
   RSpec::Matchers.define :have_hits do |expected|
     match do |actual|
       expected == actual['hits']['total']['value']

data/spec/unit/outputs/opensearch/http_client_spec.rb

@@ -157,7 +157,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
     }
 
     it "should call index template" do
-      expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
+      expect(subject.pool).to receive(:put).with("/_template/#{template_name}", nil, anything).and_return(get_response)
       subject.template_put(template_name, template)
     end
   end

data/spec/unit/outputs/opensearch/template_manager_spec.rb

@@ -13,9 +13,28 @@ require "logstash/outputs/opensearch/template_manager"
 describe LogStash::Outputs::OpenSearch::TemplateManager do
 
   describe ".default_template_path" do
-    context 'when ECS v1 is requested' do
-      it 'resolves' do
-        expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/7x.json")
+    [1, 2].each do |major_version|
+      context "when ECS is disabled with OpenSearch #{major_version}.x" do
+        it 'resolves' do
+          expect(described_class.default_template_path(major_version)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
+        end
+        it 'resolves' do
+          expect(described_class.default_template_path(major_version, :disabled)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
+        end
+      end
+    end
+    [7, 1, 2].each do |major_version|
+      context "when ECS v1 is requested with OpenSearch #{major_version}.x" do
+        it 'resolves' do
+          expect(described_class.default_template_path(major_version, :v1)).to end_with("/templates/ecs-v1/#{major_version}x.json")
+        end
+      end
+    end
+    [1, 2].each do |major_version|
+      context "when ECS v8 is requested with OpenSearch #{major_version}.x" do
+        it 'resolves' do
+          expect(described_class.default_template_path(major_version, :v8)).to end_with("/templates/ecs-v8/#{major_version}x.json")
+        end
       end
     end
   end
@@ -33,4 +52,4 @@ describe LogStash::Outputs::OpenSearch::TemplateManager do
       end
     end
   end
-end
+end

data/spec/unit/outputs/opensearch_spec.rb

@@ -12,6 +12,7 @@ require "base64"
 require "flores/random"
 require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/opensearch"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
 describe LogStash::Outputs::OpenSearch do
   subject(:opensearch_output_instance) { described_class.new(options) }
@@ -784,6 +785,21 @@ describe LogStash::Outputs::OpenSearch do
     end
   end
 
+  describe 'ecs_compatibility support', :ecs_compatibility_support do
+    [:disabled, :v1, :v8].each do |ecs_compatibility|
+      context "when `ecs_compatibility => #{ecs_compatibility}`" do
+        let(:options) { Hash.new }
+        subject(:output) { described_class.new(options.merge("ecs_compatibility" => "#{ecs_compatibility}")) }
+        context 'when registered' do
+          before(:each) { output.register }
+          it 'has the correct effective ECS compatibility setting' do
+            expect(output.ecs_compatibility).to eq(ecs_compatibility)
+          end
+        end
+      end
+    end
+  end
+  
   @private
 
   def stub_manticore_client!(manticore_double = nil)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: java
 authors:
 - Elastic
@@ -13,7 +13,7 @@ cert_chain:
   -----BEGIN CERTIFICATE-----
   MIIDfDCCAmSgAwIBAgIBATANBgkqhkiG9w0BAQUFADBCMRMwEQYDVQQDDApvcGVu
   c2VhcmNoMRYwFAYKCZImiZPyLGQBGRYGYW1hem9uMRMwEQYKCZImiZPyLGQBGRYD
-  Y29tMB4XDTIxMDgwMjIxMDQwM1oXDTIyMDgwMjIxMDQwM1owQjETMBEGA1UEAwwK
+  Y29tMB4XDTIyMDgxNzE3NTIzNFoXDTIzMDgxNzE3NTIzNFowQjETMBEGA1UEAwwK
   b3BlbnNlYXJjaDEWMBQGCgmSJomT8ixkARkWBmFtYXpvbjETMBEGCgmSJomT8ixk
   ARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1z3/jcitjV
   umXwRFn+JSBBd36qZB54Dtucf6+E2fmNPzBRhgYN5XJy/+clQJ9NIJV7C8H52P3V
@@ -24,14 +24,14 @@ cert_chain:
   zfR37/NQFkECAwEAAaN9MHswCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0O
   BBYEFJJ2myhLXK742btavNbG0IWrMNBIMCAGA1UdEQQZMBeBFW9wZW5zZWFyY2hA
   YW1hem9uLmNvbTAgBgNVHRIEGTAXgRVvcGVuc2VhcmNoQGFtYXpvbi5jb20wDQYJ
-  KoZIhvcNAQEFBQADggEBAE7gBP5ecTtKb04qmEsnbJ6+yn0LUSmxPabFBnB6h1+T
-  XW8BvBw9MpE//5fQf4HSia3m9XjRpl4WxBcJiyfLER64tk/c1JLhV2+rq3CCV/be
-  DFSP6gY93kK7jwauajGQvyHzORaW1TBM6diIRYCMLY7Isf+PTHl0xhZZBSVm8wl6
-  IstV+mTP2KC1l7hSzUDb4rrOSnpRB7AEczcDdkjwzHKIlw8rcL+PLLnFTOgqKyq3
-  yXikuH6LEVykA8pgOcB9gKsB2/zMd2ZlSj2monM8Qw9EfB14ZSDTYS8VYuwWCeF0
-  eFmXXk0ufQFKl1Yll7quHkmQ0PzKkvXTpONBT6qPkXE=
+  KoZIhvcNAQEFBQADggEBAH5pWLYwKWFh1OjdCReGz/VEAiF4jXXputoN5Z8ga+1Z
+  lg8/diJf0PlP2B46PdmxH/TVc/o+qglNO2cHVEp8xZfEd83dfioOBeK90URQUpC5
+  UZmO0LZusg46SQKwKa2ukpIy2fNi3PeHRiV+W2Zv69GoWppyLun+fMez7wVoah2r
+  r5ROUYuAvFUvga1Vm+49pKiPM5n+MAYP5t/vWhgymY3SYQ1TfewkvKAFiFXikOR+
+  r+j7FLyKuk5DzIxiCp8QN5dU71BbGUmsHf/C5UV76WLPOFX/szeaHhPwpjR3sK7r
+  5zLgCV1KP7cgDdCYMlmZGeSViU8NV+Yy8/ghrzGpqVw=
   -----END CERTIFICATE-----
-date: 2021-12-16 00:00:00.000000000 Z
+date: 2022-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -56,23 +56,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.17
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
   name: stud
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.17
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -110,23 +110,43 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2'
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.11.632
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
   name: aws-sdk
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.11.632
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.11.632
+        version: 2.3.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  name: json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -186,17 +206,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   name: opensearch-ruby
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gem. This gem
   is not a stand-alone program
@@ -224,7 +244,11 @@ files:
 - lib/logstash/outputs/opensearch/http_client_builder.rb
 - lib/logstash/outputs/opensearch/template_manager.rb
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/7x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json
 - lib/logstash/plugin_mixins/opensearch/api_configs.rb
 - lib/logstash/plugin_mixins/opensearch/common.rb
 - logstash-output-opensearch.gemspec
@@ -282,8 +306,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.10
+rubygems_version: 3.3.20
 signing_key:
 specification_version: 4
 summary: Stores logs in OpenSearch