logstash-output-opensearch 1.0.0-java → 1.3.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/COMPATIBILITY.md +27 -0
  4. data/CONTRIBUTING.md +27 -5
  5. data/DEVELOPER_GUIDE.md +16 -7
  6. data/Gemfile +1 -1
  7. data/MAINTAINERS.md +3 -0
  8. data/NOTICE +9 -2
  9. data/README.md +75 -5
  10. data/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb +101 -4
  11. data/lib/logstash/outputs/opensearch/http_client/pool.rb +3 -16
  12. data/lib/logstash/outputs/opensearch/http_client.rb +12 -23
  13. data/lib/logstash/outputs/opensearch/http_client_builder.rb +5 -2
  14. data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json +44 -0
  15. data/lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json +5252 -0
  16. data/lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json +5252 -0
  17. data/lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json +5252 -0
  18. data/lib/logstash/outputs/opensearch.rb +4 -5
  19. data/lib/logstash/plugin_mixins/opensearch/api_configs.rb +26 -0
  20. data/lib/logstash/plugin_mixins/opensearch/common.rb +2 -2
  21. data/logstash-output-opensearch.gemspec +20 -4
  22. data/spec/integration/outputs/compressed_indexing_spec.rb +11 -5
  23. data/spec/integration/outputs/create_spec.rb +7 -7
  24. data/spec/integration/outputs/delete_spec.rb +8 -8
  25. data/spec/integration/outputs/index_spec.rb +54 -12
  26. data/spec/integration/outputs/index_version_spec.rb +11 -11
  27. data/spec/integration/outputs/ingest_pipeline_spec.rb +3 -4
  28. data/spec/integration/outputs/metrics_spec.rb +0 -2
  29. data/spec/integration/outputs/no_opensearch_on_startup_spec.rb +0 -1
  30. data/spec/integration/outputs/painless_update_spec.rb +10 -10
  31. data/spec/integration/outputs/parent_spec.rb +2 -2
  32. data/spec/integration/outputs/retry_spec.rb +2 -2
  33. data/spec/integration/outputs/sniffer_spec.rb +2 -2
  34. data/spec/integration/outputs/templates_spec.rb +83 -59
  35. data/spec/integration/outputs/update_spec.rb +14 -14
  36. data/spec/opensearch_spec_helper.rb +12 -2
  37. data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb +74 -4
  38. data/spec/unit/outputs/opensearch/http_client/pool_spec.rb +4 -87
  39. data/spec/unit/outputs/opensearch/http_client_spec.rb +6 -5
  40. data/spec/unit/outputs/opensearch/template_manager_spec.rb +23 -4
  41. data/spec/unit/outputs/opensearch_spec.rb +18 -2
  42. data.tar.gz.sig +0 -0
  43. metadata +75 -19
  44. metadata.gz.sig +0 -0
  45. data/lib/logstash/outputs/opensearch/distribution_checker.rb +0 -44
  46. data/lib/logstash/plugin_mixins/opensearch/noop_distribution_checker.rb +0 -18
@@ -46,14 +46,14 @@ describe "failures in bulk class expected behavior", :integration => true do
46
46
  "template_overwrite" => true,
47
47
  "hosts" => get_host_port(),
48
48
  "retry_max_interval" => 64,
49
- "retry_initial_interval" => 2
49
+ "retry_initial_interval" => 2,
50
+ "ecs_compatibility" => "disabled" # specs are tightly tied to non-ECS defaults
50
51
  }
51
52
  next LogStash::Outputs::OpenSearch.new(settings)
52
53
  end
53
54
 
54
55
  before :each do
55
56
  # Delete all templates first.
56
- require "elasticsearch"
57
57
  allow(Stud).to receive(:stoppable_sleep)
58
58
 
59
59
  # Clean OpenSearch of data before we start.
@@ -15,9 +15,9 @@ require "socket"
15
15
  describe "pool sniffer", :integration => true do
16
16
  let(:logger) { Cabin::Channel.get }
17
17
  let(:adapter) { LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter.new(logger) }
18
- let(:es_host) { get_host_port.split(":").first }
18
+ let(:opensearch_host) { get_host_port.split(":").first }
19
19
  let(:es_port) { get_host_port.split(":").last }
20
- let(:es_ip) { IPSocket.getaddress(es_host) }
20
+ let(:opensearch_ip) { IPSocket.getaddress(opensearch_host) }
21
21
  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
22
22
  let(:options) do
23
23
  {
@@ -10,8 +10,12 @@
10
10
  require_relative "../../../spec/opensearch_spec_helper"
11
11
 
12
12
  describe "index template expected behavior", :integration => true do
13
+ let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
14
+
13
15
  subject! do
14
16
  require "logstash/outputs/opensearch"
17
+ allow_any_instance_of(LogStash::Outputs::OpenSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
18
+
15
19
  settings = {
16
20
  "manage_template" => true,
17
21
  "template_overwrite" => true,
@@ -21,8 +25,6 @@ describe "index template expected behavior", :integration => true do
21
25
  end
22
26
 
23
27
  before :each do
24
- # Delete all templates first.
25
- require "elasticsearch"
26
28
 
27
29
  # Clean OpenSearch of data before we start.
28
30
  @client = get_client
@@ -31,75 +33,97 @@ describe "index template expected behavior", :integration => true do
31
33
  # This can fail if there are no indexes, ignore failure.
32
34
  @client.indices.delete(:index => "*") rescue nil
33
35
 
34
- subject.register
35
-
36
- subject.multi_receive([
37
- LogStash::Event.new("message" => "sample message here"),
38
- LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
39
- LogStash::Event.new("somevalue" => 100),
40
- LogStash::Event.new("somevalue" => 10),
41
- LogStash::Event.new("somevalue" => 1),
42
- LogStash::Event.new("country" => "us"),
43
- LogStash::Event.new("country" => "at"),
44
- LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
45
- ])
46
-
47
- @client.indices.refresh
48
-
49
- # Wait or fail until everything's indexed.
50
- Stud::try(20.times) do
51
- r = @client.search(index: 'logstash-*')
52
- expect(r).to have_hits(8)
53
- end
54
36
  end
55
37
 
56
- it "permits phrase searching on string fields" do
57
- results = @client.search(:q => "message:\"sample message\"")
58
- expect(results).to have_hits(1)
59
- expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
60
- end
38
+ context 'with ecs_compatibility => disabled' do
39
+ let(:ecs_compatibility) { :disabled }
40
+
41
+ before :each do
42
+ subject.register
43
+
44
+ subject.multi_receive([
45
+ LogStash::Event.new("message" => "sample message here"),
46
+ LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
47
+ LogStash::Event.new("somevalue" => 100),
48
+ LogStash::Event.new("somevalue" => 10),
49
+ LogStash::Event.new("somevalue" => 1),
50
+ LogStash::Event.new("country" => "us"),
51
+ LogStash::Event.new("country" => "at"),
52
+ LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
53
+ ])
54
+
55
+ @client.indices.refresh
56
+
57
+ # Wait or fail until everything's indexed.
58
+ Stud::try(20.times) do
59
+ r = @client.search(index: 'logstash-*')
60
+ expect(r).to have_hits(8)
61
+ end
62
+ end
61
63
 
62
- it "numbers dynamically map to a numeric type and permit range queries" do
63
- results = @client.search(:q => "somevalue:[5 TO 105]")
64
- expect(results).to have_hits(2)
64
+ it "permits phrase searching on string fields" do
65
+ results = @client.search(:q => "message:\"sample message\"")
66
+ expect(results).to have_hits(1)
67
+ expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
68
+ end
65
69
 
66
- values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
67
- expect(values).to include(10)
68
- expect(values).to include(100)
69
- expect(values).to_not include(1)
70
- end
70
+ it "numbers dynamically map to a numeric type and permit range queries" do
71
+ results = @client.search(:q => "somevalue:[5 TO 105]")
72
+ expect(results).to have_hits(2)
71
73
 
72
- it "does not create .keyword field for top-level message field" do
73
- results = @client.search(:q => "message.keyword:\"sample message here\"")
74
- expect(results).to have_hits(0)
75
- end
74
+ values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
75
+ expect(values).to include(10)
76
+ expect(values).to include(100)
77
+ expect(values).to_not include(1)
78
+ end
76
79
 
77
- it "creates .keyword field for nested message fields" do
78
- results = @client.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
79
- expect(results).to have_hits(1)
80
- end
80
+ it "does not create .keyword field for top-level message field" do
81
+ results = @client.search(:q => "message.keyword:\"sample message here\"")
82
+ expect(results).to have_hits(0)
83
+ end
81
84
 
82
- it "creates .keyword field from any string field which is not_analyzed" do
83
- results = @client.search(:q => "country.keyword:\"us\"")
84
- expect(results).to have_hits(1)
85
- expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
85
+ it "creates .keyword field for nested message fields" do
86
+ results = @client.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
87
+ expect(results).to have_hits(1)
88
+ end
86
89
 
87
- # partial or terms should not work.
88
- results = @client.search(:q => "country.keyword:\"u\"")
89
- expect(results).to have_hits(0)
90
- end
90
+ it "creates .keyword field from any string field which is not_analyzed" do
91
+ results = @client.search(:q => "country.keyword:\"us\"")
92
+ expect(results).to have_hits(1)
93
+ expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
94
+
95
+ # partial or terms should not work.
96
+ results = @client.search(:q => "country.keyword:\"u\"")
97
+ expect(results).to have_hits(0)
98
+ end
99
+
100
+ it "make [geoip][location] a geo_point" do
101
+ expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
102
+ end
91
103
 
92
- it "make [geoip][location] a geo_point" do
93
- expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
104
+ it "aggregate .keyword results correctly " do
105
+ results = @client.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
106
+ terms = results["buckets"].collect { |b| b["key"] }
107
+
108
+ expect(terms).to include("us")
109
+
110
+ # 'at' is a stopword, make sure stopwords are not ignored.
111
+ expect(terms).to include("at")
112
+ end
94
113
  end
95
114
 
96
- it "aggregate .keyword results correctly " do
97
- results = @client.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
98
- terms = results["buckets"].collect { |b| b["key"] }
115
+ context 'with ECS enabled' do
116
+ let(:ecs_compatibility) { :v1 }
99
117
 
100
- expect(terms).to include("us")
118
+ before(:each) do
119
+ subject.register # should load template?
120
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
121
+ end
101
122
 
102
- # 'at' is a stopword, make sure stopwords are not ignored.
103
- expect(terms).to include("at")
123
+ it 'loads the templates' do
124
+ aggregate_failures do
125
+ expect(@client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
126
+ end
127
+ end
104
128
  end
105
129
  end
@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
12
12
  describe "Update actions without scripts", :integration => true do
13
13
  require "logstash/outputs/opensearch"
14
14
 
15
- def get_es_output( options={} )
15
+ def get_output( options={} )
16
16
  settings = {
17
17
  "manage_template" => true,
18
18
  "index" => "logstash-update",
@@ -40,20 +40,20 @@ describe "Update actions without scripts", :integration => true do
40
40
  end
41
41
 
42
42
  it "should fail without a document_id" do
43
- subject = get_es_output
43
+ subject = get_output
44
44
  expect { subject.register }.to raise_error(LogStash::ConfigurationError)
45
45
  end
46
46
 
47
47
  context "when update only" do
48
48
  it "should not create new document" do
49
- subject = get_es_output({ 'document_id' => "456" } )
49
+ subject = get_output({ 'document_id' => "456" } )
50
50
  subject.register
51
51
  subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
52
- expect {@client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
52
+ expect {@client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(OpenSearch::Transport::Transport::Errors::NotFound)
53
53
  end
54
54
 
55
55
  it "should update existing document" do
56
- subject = get_es_output({ 'document_id' => "123" })
56
+ subject = get_output({ 'document_id' => "123" })
57
57
  subject.register
58
58
  subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
59
59
  r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -63,7 +63,7 @@ describe "Update actions without scripts", :integration => true do
63
63
  # The es ruby client treats the data field differently. Make sure this doesn't
64
64
  # raise an exception
65
65
  it "should update an existing document that has a 'data' field" do
66
- subject = get_es_output({ 'document_id' => "123" })
66
+ subject = get_output({ 'document_id' => "123" })
67
67
  subject.register
68
68
  subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
69
69
  r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -72,27 +72,27 @@ describe "Update actions without scripts", :integration => true do
72
72
  end
73
73
 
74
74
  it "should allow default (internal) version" do
75
- subject = get_es_output({ 'document_id' => "123", "version" => "99" })
75
+ subject = get_output({ 'document_id' => "123", "version" => "99" })
76
76
  subject.register
77
77
  end
78
78
 
79
79
  it "should allow internal version" do
80
- subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
80
+ subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
81
81
  subject.register
82
82
  end
83
83
 
84
84
  it "should not allow external version" do
85
- subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
85
+ subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
86
86
  expect { subject.register }.to raise_error(LogStash::ConfigurationError)
87
87
  end
88
88
 
89
89
  it "should not allow external_gt version" do
90
- subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
90
+ subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
91
91
  expect { subject.register }.to raise_error(LogStash::ConfigurationError)
92
92
  end
93
93
 
94
94
  it "should not allow external_gte version" do
95
- subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
95
+ subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
96
96
  expect { subject.register }.to raise_error(LogStash::ConfigurationError)
97
97
  end
98
98
 
@@ -100,7 +100,7 @@ describe "Update actions without scripts", :integration => true do
100
100
 
101
101
  context "when update with upsert" do
102
102
  it "should create new documents with provided upsert" do
103
- subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
103
+ subject = get_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
104
104
  subject.register
105
105
  subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
106
106
  r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -108,7 +108,7 @@ describe "Update actions without scripts", :integration => true do
108
108
  end
109
109
 
110
110
  it "should create new documents with event/doc as upsert" do
111
- subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
111
+ subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true })
112
112
  subject.register
113
113
  subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
114
114
  r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -116,7 +116,7 @@ describe "Update actions without scripts", :integration => true do
116
116
  end
117
117
 
118
118
  it "should fail on documents with event/doc as upsert at external version" do
119
- subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
119
+ subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
120
120
  expect { subject.register }.to raise_error(LogStash::ConfigurationError)
121
121
  end
122
122
  end
@@ -9,7 +9,7 @@
9
9
 
10
10
  require_relative './spec_helper'
11
11
 
12
- require 'elasticsearch'
12
+ require 'opensearch'
13
13
 
14
14
  require 'json'
15
15
  require 'cabin'
@@ -24,7 +24,7 @@ module OpenSearchHelper
24
24
  end
25
25
 
26
26
  def get_client
27
- Elasticsearch::Client.new(:hosts => [get_host_port])
27
+ OpenSearch::Client.new(:hosts => [get_host_port])
28
28
  end
29
29
 
30
30
  def doc_type
@@ -49,6 +49,16 @@ module OpenSearchHelper
49
49
  RSpec.configuration.filter[:version]
50
50
  end
51
51
 
52
+ def self.check_version?(*requirement)
53
+ version = self.version
54
+ if version.nil? || version.empty?
55
+ puts "version tag isn't set. Returning false from 'check_version?`."
56
+ return false
57
+ end
58
+ release_version = Gem::Version.new(version).release
59
+ Gem::Requirement.new(requirement).satisfied_by?(release_version)
60
+ end
61
+
52
62
  RSpec::Matchers.define :have_hits do |expected|
53
63
  match do |actual|
54
64
  expected == actual['hits']['total']['value']
@@ -25,13 +25,13 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
25
25
  it "should implement host unreachable exceptions" do
26
26
  expect(subject.host_unreachable_exceptions).to be_a(Array)
27
27
  end
28
-
28
+
29
29
  describe "auth" do
30
30
  let(:user) { "myuser" }
31
31
  let(:password) { "mypassword" }
32
32
  let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
33
33
  let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
34
-
34
+
35
35
  it "should convert the auth to params" do
36
36
  resp = double("response")
37
37
  allow(resp).to receive(:call)
@@ -39,10 +39,10 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
39
39
 
40
40
  expected_uri = noauth_uri.clone
41
41
  expected_uri.path = "/"
42
-
42
+
43
43
  expect(subject.manticore).to receive(:get).
44
44
  with(expected_uri.to_s, {
45
- :headers => {"Content-Type" => "application/json"},
45
+ :headers => {"content-type" => "application/json"},
46
46
  :auth => {
47
47
  :user => user,
48
48
  :password => password,
@@ -54,6 +54,76 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
54
54
  end
55
55
  end
56
56
 
57
+ describe "aws_iam" do
58
+ let(:options) { {
59
+ :auth_type => {
60
+ "type"=>"aws_iam",
61
+ "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
62
+ "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
63
+ } }
64
+ subject { described_class.new(logger, options) }
65
+ let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
66
+ let(:sign_aws_request) { }
67
+
68
+ it "should validate AWS IAM credentials initialization" do
69
+ expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
70
+ end
71
+
72
+ it "should validate signing aws request" do
73
+ resp = double("response")
74
+ allow(resp).to receive(:call)
75
+ allow(resp).to receive(:code).and_return(200)
76
+ allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
77
+
78
+ expected_uri = uri.clone
79
+ expected_uri.path = "/"
80
+
81
+ expect(subject.manticore).to receive(:get).
82
+ with(expected_uri.to_s, {
83
+ :headers => {"content-type"=> "application/json"}
84
+ }
85
+ ).and_return resp
86
+
87
+ expect(subject).to receive(:sign_aws_request)
88
+ subject.perform_request(uri, :get, "/")
89
+ end
90
+ end
91
+
92
+ describe "basic_auth" do
93
+ let(:options) { {
94
+ :auth_type => {
95
+ "type"=>"basic",
96
+ "user" => "myuser",
97
+ "password" => "mypassword"}
98
+ } }
99
+ subject { described_class.new(logger, options) }
100
+ let(:user) {options[:auth_type]["user"]}
101
+ let(:password) {options[:auth_type]["password"]}
102
+ let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
103
+ let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
104
+
105
+ it "should validate master credentials with type as 'basic_auth'" do
106
+ resp = double("response")
107
+ allow(resp).to receive(:call)
108
+ allow(resp).to receive(:code).and_return(200)
109
+
110
+ expected_uri = noauth_uri.clone
111
+ expected_uri.path = "/"
112
+
113
+ expect(subject.manticore).to receive(:get).
114
+ with(expected_uri.to_s, {
115
+ :headers => {"content-type" => "application/json"},
116
+ :auth => {
117
+ :user => user,
118
+ :password => password,
119
+ :eager => true
120
+ }
121
+ }).and_return resp
122
+
123
+ subject.perform_request(uri, :get, "/")
124
+ end
125
+ end
126
+
57
127
  describe "bad response codes" do
58
128
  let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
59
129
 
@@ -16,8 +16,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient::Pool do
16
16
  let(:adapter) { LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter.new(logger) }
17
17
  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
18
18
  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
19
- let(:node_versions) { [ "7.0.0" ] }
20
- let(:get_distribution) { "oss" }
19
+ let(:node_versions) { [ "0.0.0" ] }
21
20
 
22
21
  subject { described_class.new(logger, adapter, initial_urls, options) }
23
22
 
@@ -32,7 +31,6 @@ describe LogStash::Outputs::OpenSearch::HttpClient::Pool do
32
31
  allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
33
32
 
34
33
  allow(subject).to receive(:get_version).with(any_args).and_return(*node_versions)
35
- allow(subject.distribution_checker).to receive(:get_distribution).and_return(get_distribution)
36
34
  end
37
35
 
38
36
  after do
@@ -212,94 +210,13 @@ describe LogStash::Outputs::OpenSearch::HttpClient::Pool do
212
210
  end
213
211
 
214
212
  it "picks the largest major version" do
215
- expect(subject.maximum_seen_major_version).to eq(7)
213
+ expect(subject.maximum_seen_major_version).to eq(0)
216
214
  end
217
215
 
218
216
  context "if there are nodes with multiple major versions" do
219
- let(:node_versions) { [ "0.0.0", "7.0.0" ] }
217
+ let(:node_versions) { [ "0.0.0", "6.0.0" ] }
220
218
  it "picks the largest major version" do
221
- expect(subject.maximum_seen_major_version).to eq(7)
222
- end
223
- end
224
- end
225
- describe "distribution checking" do
226
- before(:each) do
227
- allow(subject).to receive(:health_check_request)
228
- end
229
-
230
- let(:options) do
231
- super().merge(:distribution_checker => distribution_checker)
232
- end
233
-
234
- context 'when DistributionChecker#is_supported? returns false' do
235
- let(:distribution_checker) { double('DistributionChecker', :is_supported? => false) }
236
-
237
- it 'does not mark the URL as active' do
238
- subject.update_initial_urls
239
- expect(subject.alive_urls_count).to eq(0)
240
- end
241
- end
242
-
243
- context 'when DistributionChecker#is_supported? returns true' do
244
- let(:distribution_checker) { double('DistributionChecker', :is_supported? => true) }
245
-
246
- it 'marks the URL as active' do
247
- subject.update_initial_urls
248
- expect(subject.alive_urls_count).to eq(1)
249
- end
250
- end
251
- end
252
- describe 'distribution checking with cluster output' do
253
- let(:options) do
254
- super().merge(:distribution_checker => LogStash::Outputs::OpenSearch::DistributionChecker.new(logger))
255
- end
256
-
257
- before(:each) do
258
- allow(subject).to receive(:health_check_request)
259
- end
260
-
261
- context 'when using opensearch' do
262
-
263
- context "if cluster doesn't return a valid distribution" do
264
- let(:get_distribution) { nil }
265
-
266
- it 'marks the url as dead' do
267
- subject.update_initial_urls
268
- expect(subject.alive_urls_count).to eq(0)
269
- end
270
-
271
- it 'logs message' do
272
- expect(subject.distribution_checker).to receive(:log_not_supported).once.and_call_original
273
- subject.update_initial_urls
274
- end
275
- end
276
-
277
- context 'if cluster returns opensearch' do
278
- let(:get_distribution) { 'opensearch' }
279
-
280
- it "marks the url as active" do
281
- subject.update_initial_urls
282
- expect(subject.alive_urls_count).to eq(1)
283
- end
284
-
285
- it 'does not log message' do
286
- expect(subject.distribution_checker).to_not receive(:log_not_supported)
287
- subject.update_initial_urls
288
- end
289
- end
290
-
291
- context 'if cluster returns oss' do
292
- let(:get_distribution) { 'oss' }
293
-
294
- it 'marks the url as active' do
295
- subject.update_initial_urls
296
- expect(subject.alive_urls_count).to eq(1)
297
- end
298
-
299
- it 'does not log message' do
300
- expect(subject.distribution_checker).to_not receive(:log_not_supported)
301
- subject.update_initial_urls
302
- end
219
+ expect(subject.maximum_seen_major_version).to eq(6)
303
220
  end
304
221
  end
305
222
  end
@@ -17,6 +17,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
17
17
  opts = {
18
18
  :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
19
19
  :logger => Cabin::Channel.get,
20
+ :target_bulk_bytes => 9_000_000,
20
21
  :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy)
21
22
  }
22
23
 
@@ -156,7 +157,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
156
157
  }
157
158
 
158
159
  it "should call index template" do
159
- expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
160
+ expect(subject.pool).to receive(:put).with("/_template/#{template_name}", nil, anything).and_return(get_response)
160
161
  subject.template_put(template_name, template)
161
162
  end
162
163
  end
@@ -226,8 +227,8 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
226
227
  end
227
228
  end
228
229
 
229
- context "if a message is over TARGET_BULK_BYTES" do
230
- let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
230
+ context "if a message is over target_bulk_bytes" do
231
+ let(:target_bulk_bytes) { subject.target_bulk_bytes }
231
232
  let(:message) { "a" * (target_bulk_bytes + 1) }
232
233
 
233
234
  it "should be handled properly" do
@@ -256,8 +257,8 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
256
257
  s = subject.send(:bulk, actions)
257
258
  end
258
259
 
259
- context "if one exceeds TARGET_BULK_BYTES" do
260
- let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
260
+ context "if one exceeds target_bulk_bytes" do
261
+ let(:target_bulk_bytes) { subject.target_bulk_bytes }
261
262
  let(:message1) { "a" * (target_bulk_bytes + 1) }
262
263
  it "executes two bulk_send operations" do
263
264
  allow(subject).to receive(:join_bulk_responses)
@@ -13,9 +13,28 @@ require "logstash/outputs/opensearch/template_manager"
13
13
  describe LogStash::Outputs::OpenSearch::TemplateManager do
14
14
 
15
15
  describe ".default_template_path" do
16
- context 'when ECS v1 is requested' do
17
- it 'resolves' do
18
- expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/7x.json")
16
+ [1, 2].each do |major_version|
17
+ context "when ECS is disabled with OpenSearch #{major_version}.x" do
18
+ it 'resolves' do
19
+ expect(described_class.default_template_path(major_version)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
20
+ end
21
+ it 'resolves' do
22
+ expect(described_class.default_template_path(major_version, :disabled)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
23
+ end
24
+ end
25
+ end
26
+ [7, 1, 2].each do |major_version|
27
+ context "when ECS v1 is requested with OpenSearch #{major_version}.x" do
28
+ it 'resolves' do
29
+ expect(described_class.default_template_path(major_version, :v1)).to end_with("/templates/ecs-v1/#{major_version}x.json")
30
+ end
31
+ end
32
+ end
33
+ [1, 2].each do |major_version|
34
+ context "when ECS v8 is requested with OpenSearch #{major_version}.x" do
35
+ it 'resolves' do
36
+ expect(described_class.default_template_path(major_version, :v8)).to end_with("/templates/ecs-v8/#{major_version}x.json")
37
+ end
19
38
  end
20
39
  end
21
40
  end
@@ -33,4 +52,4 @@ describe LogStash::Outputs::OpenSearch::TemplateManager do
33
52
  end
34
53
  end
35
54
  end
36
- end
55
+ end