logstash-output-opensearch 1.3.0-java → 2.0.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/COMPATIBILITY.md +5 -3
- data/README.md +51 -34
- data/docs/ecs_compatibility.md +42 -0
- data/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb +27 -12
- data/lib/logstash/outputs/opensearch/http_client/pool.rb +11 -2
- data/lib/logstash/outputs/opensearch/http_client.rb +10 -3
- data/lib/logstash/outputs/opensearch/http_client_builder.rb +4 -2
- data/lib/logstash/outputs/opensearch/template_manager.rb +6 -5
- data/lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json +66 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json +66 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json +66 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/1x.json +3629 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/1x_index.json +3631 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/2x.json +3629 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/2x_index.json +3631 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/7x.json +3629 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v1/7x_index.json +3631 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json +5254 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json +5254 -0
- data/lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json +5254 -0
- data/lib/logstash/outputs/opensearch.rb +7 -0
- data/logstash-output-opensearch.gemspec +2 -2
- data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb +13 -0
- data/spec/unit/outputs/opensearch/http_client_spec.rb +20 -0
- data/spec/unit/outputs/opensearch/template_manager_spec.rb +8 -20
- data.tar.gz.sig +0 -0
- metadata +31 -23
- metadata.gz.sig +0 -0
@@ -199,6 +199,13 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
|
|
199
199
|
# Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
|
200
200
|
# here like `pipeline => "%{INGEST_PIPELINE}"`
|
201
201
|
config :pipeline, :validate => :string, :default => nil
|
202
|
+
|
203
|
+
# When set to true, use legacy templates via the _template API
|
204
|
+
# When false, use index templates using the _index_template API
|
205
|
+
config :legacy_template, :validate => :boolean, :default => true
|
206
|
+
|
207
|
+
# The OpenSearch server major version to use when it's not available from the Healthcheck endpoint.
|
208
|
+
config :default_server_major_version, :validate => :number
|
202
209
|
|
203
210
|
attr_reader :client
|
204
211
|
attr_reader :default_index
|
@@ -11,7 +11,7 @@ signing_key_path = "gem-private_key.pem"
|
|
11
11
|
|
12
12
|
Gem::Specification.new do |s|
|
13
13
|
s.name = 'logstash-output-opensearch'
|
14
|
-
s.version = '
|
14
|
+
s.version = '2.0.0'
|
15
15
|
|
16
16
|
s.licenses = ['Apache-2.0']
|
17
17
|
s.summary = "Stores logs in OpenSearch"
|
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
|
|
45
45
|
s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
|
46
46
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
47
47
|
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
|
48
|
-
s.add_runtime_dependency 'aws-sdk',
|
48
|
+
s.add_runtime_dependency 'aws-sdk', '~> 3'
|
49
49
|
s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
|
50
50
|
|
51
51
|
s.add_development_dependency 'logstash-codec-plain'
|
@@ -61,12 +61,25 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
|
|
61
61
|
"aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
|
62
62
|
"aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
|
63
63
|
} }
|
64
|
+
let(:options_svc) { {
|
65
|
+
:auth_type => {
|
66
|
+
"type"=>"aws_iam",
|
67
|
+
"aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
|
68
|
+
"aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
69
|
+
"service_name"=>"svc_test"}
|
70
|
+
} }
|
64
71
|
subject { described_class.new(logger, options) }
|
65
72
|
let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
66
73
|
let(:sign_aws_request) { }
|
67
74
|
|
68
75
|
it "should validate AWS IAM credentials initialization" do
|
69
76
|
expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
|
77
|
+
expect(subject.get_service_name).to eq("es")
|
78
|
+
end
|
79
|
+
|
80
|
+
it "should validate AWS IAM service_name config" do
|
81
|
+
expect(subject.aws_iam_auth_initialization(options_svc)).not_to be_nil
|
82
|
+
expect(subject.get_service_name).to eq("svc_test")
|
70
83
|
end
|
71
84
|
|
72
85
|
it "should validate signing aws request" do
|
@@ -162,6 +162,26 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
|
|
162
162
|
end
|
163
163
|
end
|
164
164
|
|
165
|
+
describe "legacy_template" do
|
166
|
+
let(:template_name) { "logstash" }
|
167
|
+
let(:template) { {} }
|
168
|
+
let(:get_response) {
|
169
|
+
double("response", :body => {}, :code => 404)
|
170
|
+
}
|
171
|
+
[true, false].each do |legacy_template|
|
172
|
+
context "when legacy_template => #{legacy_template}" do
|
173
|
+
let(:base_options) { super().merge(:client_settings => {:legacy_template => legacy_template}) }
|
174
|
+
subject { described_class.new(base_options) }
|
175
|
+
endpoint = legacy_template ? "_template" : "_index_template"
|
176
|
+
it "should call template_endpoint #{endpoint}" do
|
177
|
+
expect(subject.pool).to receive(:head).with("/#{endpoint}/#{template_name}").and_return(get_response)
|
178
|
+
expect(subject.pool).to receive(:put).with("/#{endpoint}/#{template_name}", nil, anything).and_return(get_response)
|
179
|
+
subject.template_install(template_name, template)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
165
185
|
describe "join_bulk_responses" do
|
166
186
|
subject { described_class.new(base_options) }
|
167
187
|
|
@@ -13,27 +13,15 @@ require "logstash/outputs/opensearch/template_manager"
|
|
13
13
|
describe LogStash::Outputs::OpenSearch::TemplateManager do
|
14
14
|
|
15
15
|
describe ".default_template_path" do
|
16
|
-
[1, 2].each do |major_version|
|
17
|
-
context "when ECS is disabled with OpenSearch #{major_version}.x" do
|
18
|
-
it 'resolves' do
|
19
|
-
expect(described_class.default_template_path(major_version)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
|
20
|
-
end
|
21
|
-
it 'resolves' do
|
22
|
-
expect(described_class.default_template_path(major_version, :disabled)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
16
|
[7, 1, 2].each do |major_version|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
it 'resolves' do
|
36
|
-
expect(described_class.default_template_path(major_version, :v8)).to end_with("/templates/ecs-v8/#{major_version}x.json")
|
17
|
+
[:disabled, :v1, :v8].each do |ecs_ver|
|
18
|
+
[true, false].each do |legacy_template|
|
19
|
+
context "when ECS is #{ecs_ver} with OpenSearch #{major_version}.x legacy_template:#{legacy_template}" do
|
20
|
+
suffix = legacy_template ? "" : "_index"
|
21
|
+
it 'resolves' do
|
22
|
+
expect(described_class.default_template_path(major_version, ecs_ver, legacy_template)).to end_with("/templates/ecs-#{ecs_ver}/#{major_version}x#{suffix}.json")
|
23
|
+
end
|
24
|
+
end
|
37
25
|
end
|
38
26
|
end
|
39
27
|
end
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-opensearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
@@ -31,7 +31,7 @@ cert_chain:
|
|
31
31
|
r+j7FLyKuk5DzIxiCp8QN5dU71BbGUmsHf/C5UV76WLPOFX/szeaHhPwpjR3sK7r
|
32
32
|
5zLgCV1KP7cgDdCYMlmZGeSViU8NV+Yy8/ghrzGpqVw=
|
33
33
|
-----END CERTIFICATE-----
|
34
|
-
date: 2022-
|
34
|
+
date: 2022-09-06 00:00:00.000000000 Z
|
35
35
|
dependencies:
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
requirement: !ruby/object:Gem::Requirement
|
@@ -56,23 +56,23 @@ dependencies:
|
|
56
56
|
- !ruby/object:Gem::Dependency
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - ">="
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: 0.0.17
|
62
59
|
- - "~>"
|
63
60
|
- !ruby/object:Gem::Version
|
64
61
|
version: '0.0'
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: 0.0.17
|
65
65
|
name: stud
|
66
66
|
prerelease: false
|
67
67
|
type: :runtime
|
68
68
|
version_requirements: !ruby/object:Gem::Requirement
|
69
69
|
requirements:
|
70
|
-
- - ">="
|
71
|
-
- !ruby/object:Gem::Version
|
72
|
-
version: 0.0.17
|
73
70
|
- - "~>"
|
74
71
|
- !ruby/object:Gem::Version
|
75
72
|
version: '0.0'
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 0.0.17
|
76
76
|
- !ruby/object:Gem::Dependency
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
@@ -110,43 +110,37 @@ dependencies:
|
|
110
110
|
- !ruby/object:Gem::Dependency
|
111
111
|
requirement: !ruby/object:Gem::Requirement
|
112
112
|
requirements:
|
113
|
-
- - ">="
|
114
|
-
- !ruby/object:Gem::Version
|
115
|
-
version: 2.11.632
|
116
113
|
- - "~>"
|
117
114
|
- !ruby/object:Gem::Version
|
118
|
-
version: '
|
115
|
+
version: '3'
|
119
116
|
name: aws-sdk
|
120
117
|
prerelease: false
|
121
118
|
type: :runtime
|
122
119
|
version_requirements: !ruby/object:Gem::Requirement
|
123
120
|
requirements:
|
124
|
-
- - ">="
|
125
|
-
- !ruby/object:Gem::Version
|
126
|
-
version: 2.11.632
|
127
121
|
- - "~>"
|
128
122
|
- !ruby/object:Gem::Version
|
129
|
-
version: '
|
123
|
+
version: '3'
|
130
124
|
- !ruby/object:Gem::Dependency
|
131
125
|
requirement: !ruby/object:Gem::Requirement
|
132
126
|
requirements:
|
133
|
-
- - ">="
|
134
|
-
- !ruby/object:Gem::Version
|
135
|
-
version: 2.3.0
|
136
127
|
- - "~>"
|
137
128
|
- !ruby/object:Gem::Version
|
138
129
|
version: '2'
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: 2.3.0
|
139
133
|
name: json
|
140
134
|
prerelease: false
|
141
135
|
type: :runtime
|
142
136
|
version_requirements: !ruby/object:Gem::Requirement
|
143
137
|
requirements:
|
144
|
-
- - ">="
|
145
|
-
- !ruby/object:Gem::Version
|
146
|
-
version: 2.3.0
|
147
138
|
- - "~>"
|
148
139
|
- !ruby/object:Gem::Version
|
149
140
|
version: '2'
|
141
|
+
- - ">="
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: 2.3.0
|
150
144
|
- !ruby/object:Gem::Dependency
|
151
145
|
requirement: !ruby/object:Gem::Requirement
|
152
146
|
requirements:
|
@@ -237,6 +231,7 @@ files:
|
|
237
231
|
- README.md
|
238
232
|
- RELEASING.md
|
239
233
|
- SECURITY.md
|
234
|
+
- docs/ecs_compatibility.md
|
240
235
|
- lib/logstash/outputs/opensearch.rb
|
241
236
|
- lib/logstash/outputs/opensearch/http_client.rb
|
242
237
|
- lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb
|
@@ -244,11 +239,23 @@ files:
|
|
244
239
|
- lib/logstash/outputs/opensearch/http_client_builder.rb
|
245
240
|
- lib/logstash/outputs/opensearch/template_manager.rb
|
246
241
|
- lib/logstash/outputs/opensearch/templates/ecs-disabled/1x.json
|
242
|
+
- lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json
|
247
243
|
- lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json
|
244
|
+
- lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json
|
248
245
|
- lib/logstash/outputs/opensearch/templates/ecs-disabled/7x.json
|
246
|
+
- lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json
|
247
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v1/1x.json
|
248
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v1/1x_index.json
|
249
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v1/2x.json
|
250
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v1/2x_index.json
|
251
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v1/7x.json
|
252
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v1/7x_index.json
|
249
253
|
- lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json
|
254
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json
|
250
255
|
- lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json
|
256
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json
|
251
257
|
- lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json
|
258
|
+
- lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json
|
252
259
|
- lib/logstash/plugin_mixins/opensearch/api_configs.rb
|
253
260
|
- lib/logstash/plugin_mixins/opensearch/common.rb
|
254
261
|
- logstash-output-opensearch.gemspec
|
@@ -306,7 +313,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
306
313
|
- !ruby/object:Gem::Version
|
307
314
|
version: '0'
|
308
315
|
requirements: []
|
309
|
-
|
316
|
+
rubyforge_project:
|
317
|
+
rubygems_version: 2.7.10
|
310
318
|
signing_key:
|
311
319
|
specification_version: 4
|
312
320
|
summary: Stores logs in OpenSearch
|
metadata.gz.sig
CHANGED
Binary file
|