logstash-output-opensearch 1.0.0-java → 1.3.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/COMPATIBILITY.md +27 -0
- data/CONTRIBUTING.md +27 -5
- data/DEVELOPER_GUIDE.md +16 -7
- data/Gemfile +1 -1
- data/MAINTAINERS.md +3 -0
- data/NOTICE +9 -2
- data/README.md +75 -5
- data/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb +101 -4
- data/lib/logstash/outputs/opensearch/http_client/pool.rb +3 -16
- data/lib/logstash/outputs/opensearch/http_client.rb +12 -23
- data/lib/logstash/outputs/opensearch/http_client_builder.rb +5 -2
- data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json +44 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json +5252 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json +5252 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json +5252 -0
- data/lib/logstash/outputs/opensearch.rb +4 -5
- data/lib/logstash/plugin_mixins/opensearch/api_configs.rb +26 -0
- data/lib/logstash/plugin_mixins/opensearch/common.rb +2 -2
- data/logstash-output-opensearch.gemspec +20 -4
- data/spec/integration/outputs/compressed_indexing_spec.rb +11 -5
- data/spec/integration/outputs/create_spec.rb +7 -7
- data/spec/integration/outputs/delete_spec.rb +8 -8
- data/spec/integration/outputs/index_spec.rb +54 -12
- data/spec/integration/outputs/index_version_spec.rb +11 -11
- data/spec/integration/outputs/ingest_pipeline_spec.rb +3 -4
- data/spec/integration/outputs/metrics_spec.rb +0 -2
- data/spec/integration/outputs/no_opensearch_on_startup_spec.rb +0 -1
- data/spec/integration/outputs/painless_update_spec.rb +10 -10
- data/spec/integration/outputs/parent_spec.rb +2 -2
- data/spec/integration/outputs/retry_spec.rb +2 -2
- data/spec/integration/outputs/sniffer_spec.rb +2 -2
- data/spec/integration/outputs/templates_spec.rb +83 -59
- data/spec/integration/outputs/update_spec.rb +14 -14
- data/spec/opensearch_spec_helper.rb +12 -2
- data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb +74 -4
- data/spec/unit/outputs/opensearch/http_client/pool_spec.rb +4 -87
- data/spec/unit/outputs/opensearch/http_client_spec.rb +6 -5
- data/spec/unit/outputs/opensearch/template_manager_spec.rb +23 -4
- data/spec/unit/outputs/opensearch_spec.rb +18 -2
- data.tar.gz.sig +0 -0
- metadata +75 -19
- metadata.gz.sig +0 -0
- data/lib/logstash/outputs/opensearch/distribution_checker.rb +0 -44
- data/lib/logstash/plugin_mixins/opensearch/noop_distribution_checker.rb +0 -18
|
@@ -46,14 +46,14 @@ describe "failures in bulk class expected behavior", :integration => true do
|
|
|
46
46
|
"template_overwrite" => true,
|
|
47
47
|
"hosts" => get_host_port(),
|
|
48
48
|
"retry_max_interval" => 64,
|
|
49
|
-
"retry_initial_interval" => 2
|
|
49
|
+
"retry_initial_interval" => 2,
|
|
50
|
+
"ecs_compatibility" => "disabled" # specs are tightly tied to non-ECS defaults
|
|
50
51
|
}
|
|
51
52
|
next LogStash::Outputs::OpenSearch.new(settings)
|
|
52
53
|
end
|
|
53
54
|
|
|
54
55
|
before :each do
|
|
55
56
|
# Delete all templates first.
|
|
56
|
-
require "elasticsearch"
|
|
57
57
|
allow(Stud).to receive(:stoppable_sleep)
|
|
58
58
|
|
|
59
59
|
# Clean OpenSearch of data before we start.
|
|
@@ -15,9 +15,9 @@ require "socket"
|
|
|
15
15
|
describe "pool sniffer", :integration => true do
|
|
16
16
|
let(:logger) { Cabin::Channel.get }
|
|
17
17
|
let(:adapter) { LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter.new(logger) }
|
|
18
|
-
let(:
|
|
18
|
+
let(:opensearch_host) { get_host_port.split(":").first }
|
|
19
19
|
let(:es_port) { get_host_port.split(":").last }
|
|
20
|
-
let(:
|
|
20
|
+
let(:opensearch_ip) { IPSocket.getaddress(opensearch_host) }
|
|
21
21
|
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
|
|
22
22
|
let(:options) do
|
|
23
23
|
{
|
|
@@ -10,8 +10,12 @@
|
|
|
10
10
|
require_relative "../../../spec/opensearch_spec_helper"
|
|
11
11
|
|
|
12
12
|
describe "index template expected behavior", :integration => true do
|
|
13
|
+
let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
|
|
14
|
+
|
|
13
15
|
subject! do
|
|
14
16
|
require "logstash/outputs/opensearch"
|
|
17
|
+
allow_any_instance_of(LogStash::Outputs::OpenSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
|
18
|
+
|
|
15
19
|
settings = {
|
|
16
20
|
"manage_template" => true,
|
|
17
21
|
"template_overwrite" => true,
|
|
@@ -21,8 +25,6 @@ describe "index template expected behavior", :integration => true do
|
|
|
21
25
|
end
|
|
22
26
|
|
|
23
27
|
before :each do
|
|
24
|
-
# Delete all templates first.
|
|
25
|
-
require "elasticsearch"
|
|
26
28
|
|
|
27
29
|
# Clean OpenSearch of data before we start.
|
|
28
30
|
@client = get_client
|
|
@@ -31,75 +33,97 @@ describe "index template expected behavior", :integration => true do
|
|
|
31
33
|
# This can fail if there are no indexes, ignore failure.
|
|
32
34
|
@client.indices.delete(:index => "*") rescue nil
|
|
33
35
|
|
|
34
|
-
subject.register
|
|
35
|
-
|
|
36
|
-
subject.multi_receive([
|
|
37
|
-
LogStash::Event.new("message" => "sample message here"),
|
|
38
|
-
LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
|
|
39
|
-
LogStash::Event.new("somevalue" => 100),
|
|
40
|
-
LogStash::Event.new("somevalue" => 10),
|
|
41
|
-
LogStash::Event.new("somevalue" => 1),
|
|
42
|
-
LogStash::Event.new("country" => "us"),
|
|
43
|
-
LogStash::Event.new("country" => "at"),
|
|
44
|
-
LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
|
|
45
|
-
])
|
|
46
|
-
|
|
47
|
-
@client.indices.refresh
|
|
48
|
-
|
|
49
|
-
# Wait or fail until everything's indexed.
|
|
50
|
-
Stud::try(20.times) do
|
|
51
|
-
r = @client.search(index: 'logstash-*')
|
|
52
|
-
expect(r).to have_hits(8)
|
|
53
|
-
end
|
|
54
36
|
end
|
|
55
37
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
38
|
+
context 'with ecs_compatibility => disabled' do
|
|
39
|
+
let(:ecs_compatibility) { :disabled }
|
|
40
|
+
|
|
41
|
+
before :each do
|
|
42
|
+
subject.register
|
|
43
|
+
|
|
44
|
+
subject.multi_receive([
|
|
45
|
+
LogStash::Event.new("message" => "sample message here"),
|
|
46
|
+
LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
|
|
47
|
+
LogStash::Event.new("somevalue" => 100),
|
|
48
|
+
LogStash::Event.new("somevalue" => 10),
|
|
49
|
+
LogStash::Event.new("somevalue" => 1),
|
|
50
|
+
LogStash::Event.new("country" => "us"),
|
|
51
|
+
LogStash::Event.new("country" => "at"),
|
|
52
|
+
LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
|
|
53
|
+
])
|
|
54
|
+
|
|
55
|
+
@client.indices.refresh
|
|
56
|
+
|
|
57
|
+
# Wait or fail until everything's indexed.
|
|
58
|
+
Stud::try(20.times) do
|
|
59
|
+
r = @client.search(index: 'logstash-*')
|
|
60
|
+
expect(r).to have_hits(8)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
61
63
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
64
|
+
it "permits phrase searching on string fields" do
|
|
65
|
+
results = @client.search(:q => "message:\"sample message\"")
|
|
66
|
+
expect(results).to have_hits(1)
|
|
67
|
+
expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
|
|
68
|
+
end
|
|
65
69
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
expect(values).to_not include(1)
|
|
70
|
-
end
|
|
70
|
+
it "numbers dynamically map to a numeric type and permit range queries" do
|
|
71
|
+
results = @client.search(:q => "somevalue:[5 TO 105]")
|
|
72
|
+
expect(results).to have_hits(2)
|
|
71
73
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
74
|
+
values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
|
|
75
|
+
expect(values).to include(10)
|
|
76
|
+
expect(values).to include(100)
|
|
77
|
+
expect(values).to_not include(1)
|
|
78
|
+
end
|
|
76
79
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
it "does not create .keyword field for top-level message field" do
|
|
81
|
+
results = @client.search(:q => "message.keyword:\"sample message here\"")
|
|
82
|
+
expect(results).to have_hits(0)
|
|
83
|
+
end
|
|
81
84
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
85
|
+
it "creates .keyword field for nested message fields" do
|
|
86
|
+
results = @client.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
|
|
87
|
+
expect(results).to have_hits(1)
|
|
88
|
+
end
|
|
86
89
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
90
|
+
it "creates .keyword field from any string field which is not_analyzed" do
|
|
91
|
+
results = @client.search(:q => "country.keyword:\"us\"")
|
|
92
|
+
expect(results).to have_hits(1)
|
|
93
|
+
expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
|
|
94
|
+
|
|
95
|
+
# partial or terms should not work.
|
|
96
|
+
results = @client.search(:q => "country.keyword:\"u\"")
|
|
97
|
+
expect(results).to have_hits(0)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
it "make [geoip][location] a geo_point" do
|
|
101
|
+
expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
|
|
102
|
+
end
|
|
91
103
|
|
|
92
|
-
|
|
93
|
-
|
|
104
|
+
it "aggregate .keyword results correctly " do
|
|
105
|
+
results = @client.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
|
|
106
|
+
terms = results["buckets"].collect { |b| b["key"] }
|
|
107
|
+
|
|
108
|
+
expect(terms).to include("us")
|
|
109
|
+
|
|
110
|
+
# 'at' is a stopword, make sure stopwords are not ignored.
|
|
111
|
+
expect(terms).to include("at")
|
|
112
|
+
end
|
|
94
113
|
end
|
|
95
114
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
terms = results["buckets"].collect { |b| b["key"] }
|
|
115
|
+
context 'with ECS enabled' do
|
|
116
|
+
let(:ecs_compatibility) { :v1 }
|
|
99
117
|
|
|
100
|
-
|
|
118
|
+
before(:each) do
|
|
119
|
+
subject.register # should load template?
|
|
120
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
121
|
+
end
|
|
101
122
|
|
|
102
|
-
|
|
103
|
-
|
|
123
|
+
it 'loads the templates' do
|
|
124
|
+
aggregate_failures do
|
|
125
|
+
expect(@client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
|
|
126
|
+
end
|
|
127
|
+
end
|
|
104
128
|
end
|
|
105
129
|
end
|
|
@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
|
|
|
12
12
|
describe "Update actions without scripts", :integration => true do
|
|
13
13
|
require "logstash/outputs/opensearch"
|
|
14
14
|
|
|
15
|
-
def
|
|
15
|
+
def get_output( options={} )
|
|
16
16
|
settings = {
|
|
17
17
|
"manage_template" => true,
|
|
18
18
|
"index" => "logstash-update",
|
|
@@ -40,20 +40,20 @@ describe "Update actions without scripts", :integration => true do
|
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
it "should fail without a document_id" do
|
|
43
|
-
subject =
|
|
43
|
+
subject = get_output
|
|
44
44
|
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
context "when update only" do
|
|
48
48
|
it "should not create new document" do
|
|
49
|
-
subject =
|
|
49
|
+
subject = get_output({ 'document_id' => "456" } )
|
|
50
50
|
subject.register
|
|
51
51
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
52
|
-
expect {@client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(
|
|
52
|
+
expect {@client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(OpenSearch::Transport::Transport::Errors::NotFound)
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
it "should update existing document" do
|
|
56
|
-
subject =
|
|
56
|
+
subject = get_output({ 'document_id' => "123" })
|
|
57
57
|
subject.register
|
|
58
58
|
subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
|
|
59
59
|
r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
|
@@ -63,7 +63,7 @@ describe "Update actions without scripts", :integration => true do
|
|
|
63
63
|
# The es ruby client treats the data field differently. Make sure this doesn't
|
|
64
64
|
# raise an exception
|
|
65
65
|
it "should update an existing document that has a 'data' field" do
|
|
66
|
-
subject =
|
|
66
|
+
subject = get_output({ 'document_id' => "123" })
|
|
67
67
|
subject.register
|
|
68
68
|
subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
|
|
69
69
|
r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
|
@@ -72,27 +72,27 @@ describe "Update actions without scripts", :integration => true do
|
|
|
72
72
|
end
|
|
73
73
|
|
|
74
74
|
it "should allow default (internal) version" do
|
|
75
|
-
subject =
|
|
75
|
+
subject = get_output({ 'document_id' => "123", "version" => "99" })
|
|
76
76
|
subject.register
|
|
77
77
|
end
|
|
78
78
|
|
|
79
79
|
it "should allow internal version" do
|
|
80
|
-
subject =
|
|
80
|
+
subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
|
|
81
81
|
subject.register
|
|
82
82
|
end
|
|
83
83
|
|
|
84
84
|
it "should not allow external version" do
|
|
85
|
-
subject =
|
|
85
|
+
subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
|
|
86
86
|
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
|
87
87
|
end
|
|
88
88
|
|
|
89
89
|
it "should not allow external_gt version" do
|
|
90
|
-
subject =
|
|
90
|
+
subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
|
|
91
91
|
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
|
92
92
|
end
|
|
93
93
|
|
|
94
94
|
it "should not allow external_gte version" do
|
|
95
|
-
subject =
|
|
95
|
+
subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
|
|
96
96
|
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
|
97
97
|
end
|
|
98
98
|
|
|
@@ -100,7 +100,7 @@ describe "Update actions without scripts", :integration => true do
|
|
|
100
100
|
|
|
101
101
|
context "when update with upsert" do
|
|
102
102
|
it "should create new documents with provided upsert" do
|
|
103
|
-
subject =
|
|
103
|
+
subject = get_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
|
104
104
|
subject.register
|
|
105
105
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
106
106
|
r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
|
@@ -108,7 +108,7 @@ describe "Update actions without scripts", :integration => true do
|
|
|
108
108
|
end
|
|
109
109
|
|
|
110
110
|
it "should create new documents with event/doc as upsert" do
|
|
111
|
-
subject =
|
|
111
|
+
subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true })
|
|
112
112
|
subject.register
|
|
113
113
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
114
114
|
r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
|
@@ -116,7 +116,7 @@ describe "Update actions without scripts", :integration => true do
|
|
|
116
116
|
end
|
|
117
117
|
|
|
118
118
|
it "should fail on documents with event/doc as upsert at external version" do
|
|
119
|
-
subject =
|
|
119
|
+
subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
|
|
120
120
|
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
|
121
121
|
end
|
|
122
122
|
end
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
require_relative './spec_helper'
|
|
11
11
|
|
|
12
|
-
require '
|
|
12
|
+
require 'opensearch'
|
|
13
13
|
|
|
14
14
|
require 'json'
|
|
15
15
|
require 'cabin'
|
|
@@ -24,7 +24,7 @@ module OpenSearchHelper
|
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
def get_client
|
|
27
|
-
|
|
27
|
+
OpenSearch::Client.new(:hosts => [get_host_port])
|
|
28
28
|
end
|
|
29
29
|
|
|
30
30
|
def doc_type
|
|
@@ -49,6 +49,16 @@ module OpenSearchHelper
|
|
|
49
49
|
RSpec.configuration.filter[:version]
|
|
50
50
|
end
|
|
51
51
|
|
|
52
|
+
def self.check_version?(*requirement)
|
|
53
|
+
version = self.version
|
|
54
|
+
if version.nil? || version.empty?
|
|
55
|
+
puts "version tag isn't set. Returning false from 'check_version?`."
|
|
56
|
+
return false
|
|
57
|
+
end
|
|
58
|
+
release_version = Gem::Version.new(version).release
|
|
59
|
+
Gem::Requirement.new(requirement).satisfied_by?(release_version)
|
|
60
|
+
end
|
|
61
|
+
|
|
52
62
|
RSpec::Matchers.define :have_hits do |expected|
|
|
53
63
|
match do |actual|
|
|
54
64
|
expected == actual['hits']['total']['value']
|
|
@@ -25,13 +25,13 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
|
|
|
25
25
|
it "should implement host unreachable exceptions" do
|
|
26
26
|
expect(subject.host_unreachable_exceptions).to be_a(Array)
|
|
27
27
|
end
|
|
28
|
-
|
|
28
|
+
|
|
29
29
|
describe "auth" do
|
|
30
30
|
let(:user) { "myuser" }
|
|
31
31
|
let(:password) { "mypassword" }
|
|
32
32
|
let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
|
|
33
33
|
let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
|
|
34
|
-
|
|
34
|
+
|
|
35
35
|
it "should convert the auth to params" do
|
|
36
36
|
resp = double("response")
|
|
37
37
|
allow(resp).to receive(:call)
|
|
@@ -39,10 +39,10 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
|
|
|
39
39
|
|
|
40
40
|
expected_uri = noauth_uri.clone
|
|
41
41
|
expected_uri.path = "/"
|
|
42
|
-
|
|
42
|
+
|
|
43
43
|
expect(subject.manticore).to receive(:get).
|
|
44
44
|
with(expected_uri.to_s, {
|
|
45
|
-
:headers => {"
|
|
45
|
+
:headers => {"content-type" => "application/json"},
|
|
46
46
|
:auth => {
|
|
47
47
|
:user => user,
|
|
48
48
|
:password => password,
|
|
@@ -54,6 +54,76 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
|
|
|
54
54
|
end
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
+
describe "aws_iam" do
|
|
58
|
+
let(:options) { {
|
|
59
|
+
:auth_type => {
|
|
60
|
+
"type"=>"aws_iam",
|
|
61
|
+
"aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
|
|
62
|
+
"aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
|
|
63
|
+
} }
|
|
64
|
+
subject { described_class.new(logger, options) }
|
|
65
|
+
let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
|
66
|
+
let(:sign_aws_request) { }
|
|
67
|
+
|
|
68
|
+
it "should validate AWS IAM credentials initialization" do
|
|
69
|
+
expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
it "should validate signing aws request" do
|
|
73
|
+
resp = double("response")
|
|
74
|
+
allow(resp).to receive(:call)
|
|
75
|
+
allow(resp).to receive(:code).and_return(200)
|
|
76
|
+
allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
|
|
77
|
+
|
|
78
|
+
expected_uri = uri.clone
|
|
79
|
+
expected_uri.path = "/"
|
|
80
|
+
|
|
81
|
+
expect(subject.manticore).to receive(:get).
|
|
82
|
+
with(expected_uri.to_s, {
|
|
83
|
+
:headers => {"content-type"=> "application/json"}
|
|
84
|
+
}
|
|
85
|
+
).and_return resp
|
|
86
|
+
|
|
87
|
+
expect(subject).to receive(:sign_aws_request)
|
|
88
|
+
subject.perform_request(uri, :get, "/")
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
describe "basic_auth" do
|
|
93
|
+
let(:options) { {
|
|
94
|
+
:auth_type => {
|
|
95
|
+
"type"=>"basic",
|
|
96
|
+
"user" => "myuser",
|
|
97
|
+
"password" => "mypassword"}
|
|
98
|
+
} }
|
|
99
|
+
subject { described_class.new(logger, options) }
|
|
100
|
+
let(:user) {options[:auth_type]["user"]}
|
|
101
|
+
let(:password) {options[:auth_type]["password"]}
|
|
102
|
+
let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
|
|
103
|
+
let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
|
104
|
+
|
|
105
|
+
it "should validate master credentials with type as 'basic_auth'" do
|
|
106
|
+
resp = double("response")
|
|
107
|
+
allow(resp).to receive(:call)
|
|
108
|
+
allow(resp).to receive(:code).and_return(200)
|
|
109
|
+
|
|
110
|
+
expected_uri = noauth_uri.clone
|
|
111
|
+
expected_uri.path = "/"
|
|
112
|
+
|
|
113
|
+
expect(subject.manticore).to receive(:get).
|
|
114
|
+
with(expected_uri.to_s, {
|
|
115
|
+
:headers => {"content-type" => "application/json"},
|
|
116
|
+
:auth => {
|
|
117
|
+
:user => user,
|
|
118
|
+
:password => password,
|
|
119
|
+
:eager => true
|
|
120
|
+
}
|
|
121
|
+
}).and_return resp
|
|
122
|
+
|
|
123
|
+
subject.perform_request(uri, :get, "/")
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
57
127
|
describe "bad response codes" do
|
|
58
128
|
let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
|
59
129
|
|
|
@@ -16,8 +16,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient::Pool do
|
|
|
16
16
|
let(:adapter) { LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter.new(logger) }
|
|
17
17
|
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
|
18
18
|
let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
|
|
19
|
-
let(:node_versions) { [ "
|
|
20
|
-
let(:get_distribution) { "oss" }
|
|
19
|
+
let(:node_versions) { [ "0.0.0" ] }
|
|
21
20
|
|
|
22
21
|
subject { described_class.new(logger, adapter, initial_urls, options) }
|
|
23
22
|
|
|
@@ -32,7 +31,6 @@ describe LogStash::Outputs::OpenSearch::HttpClient::Pool do
|
|
|
32
31
|
allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
|
|
33
32
|
|
|
34
33
|
allow(subject).to receive(:get_version).with(any_args).and_return(*node_versions)
|
|
35
|
-
allow(subject.distribution_checker).to receive(:get_distribution).and_return(get_distribution)
|
|
36
34
|
end
|
|
37
35
|
|
|
38
36
|
after do
|
|
@@ -212,94 +210,13 @@ describe LogStash::Outputs::OpenSearch::HttpClient::Pool do
|
|
|
212
210
|
end
|
|
213
211
|
|
|
214
212
|
it "picks the largest major version" do
|
|
215
|
-
expect(subject.maximum_seen_major_version).to eq(
|
|
213
|
+
expect(subject.maximum_seen_major_version).to eq(0)
|
|
216
214
|
end
|
|
217
215
|
|
|
218
216
|
context "if there are nodes with multiple major versions" do
|
|
219
|
-
let(:node_versions) { [ "0.0.0", "
|
|
217
|
+
let(:node_versions) { [ "0.0.0", "6.0.0" ] }
|
|
220
218
|
it "picks the largest major version" do
|
|
221
|
-
expect(subject.maximum_seen_major_version).to eq(
|
|
222
|
-
end
|
|
223
|
-
end
|
|
224
|
-
end
|
|
225
|
-
describe "distribution checking" do
|
|
226
|
-
before(:each) do
|
|
227
|
-
allow(subject).to receive(:health_check_request)
|
|
228
|
-
end
|
|
229
|
-
|
|
230
|
-
let(:options) do
|
|
231
|
-
super().merge(:distribution_checker => distribution_checker)
|
|
232
|
-
end
|
|
233
|
-
|
|
234
|
-
context 'when DistributionChecker#is_supported? returns false' do
|
|
235
|
-
let(:distribution_checker) { double('DistributionChecker', :is_supported? => false) }
|
|
236
|
-
|
|
237
|
-
it 'does not mark the URL as active' do
|
|
238
|
-
subject.update_initial_urls
|
|
239
|
-
expect(subject.alive_urls_count).to eq(0)
|
|
240
|
-
end
|
|
241
|
-
end
|
|
242
|
-
|
|
243
|
-
context 'when DistributionChecker#is_supported? returns true' do
|
|
244
|
-
let(:distribution_checker) { double('DistributionChecker', :is_supported? => true) }
|
|
245
|
-
|
|
246
|
-
it 'marks the URL as active' do
|
|
247
|
-
subject.update_initial_urls
|
|
248
|
-
expect(subject.alive_urls_count).to eq(1)
|
|
249
|
-
end
|
|
250
|
-
end
|
|
251
|
-
end
|
|
252
|
-
describe 'distribution checking with cluster output' do
|
|
253
|
-
let(:options) do
|
|
254
|
-
super().merge(:distribution_checker => LogStash::Outputs::OpenSearch::DistributionChecker.new(logger))
|
|
255
|
-
end
|
|
256
|
-
|
|
257
|
-
before(:each) do
|
|
258
|
-
allow(subject).to receive(:health_check_request)
|
|
259
|
-
end
|
|
260
|
-
|
|
261
|
-
context 'when using opensearch' do
|
|
262
|
-
|
|
263
|
-
context "if cluster doesn't return a valid distribution" do
|
|
264
|
-
let(:get_distribution) { nil }
|
|
265
|
-
|
|
266
|
-
it 'marks the url as dead' do
|
|
267
|
-
subject.update_initial_urls
|
|
268
|
-
expect(subject.alive_urls_count).to eq(0)
|
|
269
|
-
end
|
|
270
|
-
|
|
271
|
-
it 'logs message' do
|
|
272
|
-
expect(subject.distribution_checker).to receive(:log_not_supported).once.and_call_original
|
|
273
|
-
subject.update_initial_urls
|
|
274
|
-
end
|
|
275
|
-
end
|
|
276
|
-
|
|
277
|
-
context 'if cluster returns opensearch' do
|
|
278
|
-
let(:get_distribution) { 'opensearch' }
|
|
279
|
-
|
|
280
|
-
it "marks the url as active" do
|
|
281
|
-
subject.update_initial_urls
|
|
282
|
-
expect(subject.alive_urls_count).to eq(1)
|
|
283
|
-
end
|
|
284
|
-
|
|
285
|
-
it 'does not log message' do
|
|
286
|
-
expect(subject.distribution_checker).to_not receive(:log_not_supported)
|
|
287
|
-
subject.update_initial_urls
|
|
288
|
-
end
|
|
289
|
-
end
|
|
290
|
-
|
|
291
|
-
context 'if cluster returns oss' do
|
|
292
|
-
let(:get_distribution) { 'oss' }
|
|
293
|
-
|
|
294
|
-
it 'marks the url as active' do
|
|
295
|
-
subject.update_initial_urls
|
|
296
|
-
expect(subject.alive_urls_count).to eq(1)
|
|
297
|
-
end
|
|
298
|
-
|
|
299
|
-
it 'does not log message' do
|
|
300
|
-
expect(subject.distribution_checker).to_not receive(:log_not_supported)
|
|
301
|
-
subject.update_initial_urls
|
|
302
|
-
end
|
|
219
|
+
expect(subject.maximum_seen_major_version).to eq(6)
|
|
303
220
|
end
|
|
304
221
|
end
|
|
305
222
|
end
|
|
@@ -17,6 +17,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
|
|
|
17
17
|
opts = {
|
|
18
18
|
:hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
|
|
19
19
|
:logger => Cabin::Channel.get,
|
|
20
|
+
:target_bulk_bytes => 9_000_000,
|
|
20
21
|
:metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy)
|
|
21
22
|
}
|
|
22
23
|
|
|
@@ -156,7 +157,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
|
|
|
156
157
|
}
|
|
157
158
|
|
|
158
159
|
it "should call index template" do
|
|
159
|
-
expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
|
|
160
|
+
expect(subject.pool).to receive(:put).with("/_template/#{template_name}", nil, anything).and_return(get_response)
|
|
160
161
|
subject.template_put(template_name, template)
|
|
161
162
|
end
|
|
162
163
|
end
|
|
@@ -226,8 +227,8 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
|
|
|
226
227
|
end
|
|
227
228
|
end
|
|
228
229
|
|
|
229
|
-
context "if a message is over
|
|
230
|
-
let(:target_bulk_bytes) {
|
|
230
|
+
context "if a message is over target_bulk_bytes" do
|
|
231
|
+
let(:target_bulk_bytes) { subject.target_bulk_bytes }
|
|
231
232
|
let(:message) { "a" * (target_bulk_bytes + 1) }
|
|
232
233
|
|
|
233
234
|
it "should be handled properly" do
|
|
@@ -256,8 +257,8 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
|
|
|
256
257
|
s = subject.send(:bulk, actions)
|
|
257
258
|
end
|
|
258
259
|
|
|
259
|
-
context "if one exceeds
|
|
260
|
-
let(:target_bulk_bytes) {
|
|
260
|
+
context "if one exceeds target_bulk_bytes" do
|
|
261
|
+
let(:target_bulk_bytes) { subject.target_bulk_bytes }
|
|
261
262
|
let(:message1) { "a" * (target_bulk_bytes + 1) }
|
|
262
263
|
it "executes two bulk_send operations" do
|
|
263
264
|
allow(subject).to receive(:join_bulk_responses)
|
|
@@ -13,9 +13,28 @@ require "logstash/outputs/opensearch/template_manager"
|
|
|
13
13
|
describe LogStash::Outputs::OpenSearch::TemplateManager do
|
|
14
14
|
|
|
15
15
|
describe ".default_template_path" do
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
[1, 2].each do |major_version|
|
|
17
|
+
context "when ECS is disabled with OpenSearch #{major_version}.x" do
|
|
18
|
+
it 'resolves' do
|
|
19
|
+
expect(described_class.default_template_path(major_version)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
|
|
20
|
+
end
|
|
21
|
+
it 'resolves' do
|
|
22
|
+
expect(described_class.default_template_path(major_version, :disabled)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
[7, 1, 2].each do |major_version|
|
|
27
|
+
context "when ECS v1 is requested with OpenSearch #{major_version}.x" do
|
|
28
|
+
it 'resolves' do
|
|
29
|
+
expect(described_class.default_template_path(major_version, :v1)).to end_with("/templates/ecs-v1/#{major_version}x.json")
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
[1, 2].each do |major_version|
|
|
34
|
+
context "when ECS v8 is requested with OpenSearch #{major_version}.x" do
|
|
35
|
+
it 'resolves' do
|
|
36
|
+
expect(described_class.default_template_path(major_version, :v8)).to end_with("/templates/ecs-v8/#{major_version}x.json")
|
|
37
|
+
end
|
|
19
38
|
end
|
|
20
39
|
end
|
|
21
40
|
end
|
|
@@ -33,4 +52,4 @@ describe LogStash::Outputs::OpenSearch::TemplateManager do
|
|
|
33
52
|
end
|
|
34
53
|
end
|
|
35
54
|
end
|
|
36
|
-
end
|
|
55
|
+
end
|