logstash-output-opensearch 1.3.0-java → 2.0.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/COMPATIBILITY.md +5 -3
  4. data/README.md +51 -34
  5. data/docs/ecs_compatibility.md +42 -0
  6. data/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb +27 -12
  7. data/lib/logstash/outputs/opensearch/http_client/pool.rb +11 -2
  8. data/lib/logstash/outputs/opensearch/http_client.rb +10 -3
  9. data/lib/logstash/outputs/opensearch/http_client_builder.rb +4 -2
  10. data/lib/logstash/outputs/opensearch/template_manager.rb +6 -5
  11. data/lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json +66 -0
  12. data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json +66 -0
  13. data/lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json +66 -0
  14. data/lib/logstash/outputs/opensearch/templates/ecs-v1/1x.json +3629 -0
  15. data/lib/logstash/outputs/opensearch/templates/ecs-v1/1x_index.json +3631 -0
  16. data/lib/logstash/outputs/opensearch/templates/ecs-v1/2x.json +3629 -0
  17. data/lib/logstash/outputs/opensearch/templates/ecs-v1/2x_index.json +3631 -0
  18. data/lib/logstash/outputs/opensearch/templates/ecs-v1/7x.json +3629 -0
  19. data/lib/logstash/outputs/opensearch/templates/ecs-v1/7x_index.json +3631 -0
  20. data/lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json +5254 -0
  21. data/lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json +5254 -0
  22. data/lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json +5254 -0
  23. data/lib/logstash/outputs/opensearch.rb +7 -0
  24. data/logstash-output-opensearch.gemspec +2 -2
  25. data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb +13 -0
  26. data/spec/unit/outputs/opensearch/http_client_spec.rb +20 -0
  27. data/spec/unit/outputs/opensearch/template_manager_spec.rb +8 -20
  28. data.tar.gz.sig +0 -0
  29. metadata +31 -23
  30. metadata.gz.sig +0 -0
@@ -199,6 +199,13 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
199
199
  # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
200
200
  # here like `pipeline => "%{INGEST_PIPELINE}"`
201
201
  config :pipeline, :validate => :string, :default => nil
202
+
203
+ # When set to true, use legacy templates via the _template API
204
+ # When false, use index templates using the _index_template API
205
+ config :legacy_template, :validate => :boolean, :default => true
206
+
207
+ # The OpenSearch server major version to use when it's not available from the Healthcheck endpoint.
208
+ config :default_server_major_version, :validate => :number
202
209
 
203
210
  attr_reader :client
204
211
  attr_reader :default_index
@@ -11,7 +11,7 @@ signing_key_path = "gem-private_key.pem"
11
11
 
12
12
  Gem::Specification.new do |s|
13
13
  s.name = 'logstash-output-opensearch'
14
- s.version = '1.3.0'
14
+ s.version = '2.0.0'
15
15
 
16
16
  s.licenses = ['Apache-2.0']
17
17
  s.summary = "Stores logs in OpenSearch"
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
45
45
  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
46
46
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
47
47
  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
48
- s.add_runtime_dependency 'aws-sdk', '>= 2.11.632', '~> 2'
48
+ s.add_runtime_dependency 'aws-sdk', '~> 3'
49
49
  s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
50
50
 
51
51
  s.add_development_dependency 'logstash-codec-plain'
@@ -61,12 +61,25 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
61
61
  "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
62
62
  "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
63
63
  } }
64
+ let(:options_svc) { {
65
+ :auth_type => {
66
+ "type"=>"aws_iam",
67
+ "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
68
+ "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
69
+ "service_name"=>"svc_test"}
70
+ } }
64
71
  subject { described_class.new(logger, options) }
65
72
  let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
66
73
  let(:sign_aws_request) { }
67
74
 
68
75
  it "should validate AWS IAM credentials initialization" do
69
76
  expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
77
+ expect(subject.get_service_name).to eq("es")
78
+ end
79
+
80
+ it "should validate AWS IAM service_name config" do
81
+ expect(subject.aws_iam_auth_initialization(options_svc)).not_to be_nil
82
+ expect(subject.get_service_name).to eq("svc_test")
70
83
  end
71
84
 
72
85
  it "should validate signing aws request" do
@@ -162,6 +162,26 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
162
162
  end
163
163
  end
164
164
 
165
+ describe "legacy_template" do
166
+ let(:template_name) { "logstash" }
167
+ let(:template) { {} }
168
+ let(:get_response) {
169
+ double("response", :body => {}, :code => 404)
170
+ }
171
+ [true, false].each do |legacy_template|
172
+ context "when legacy_template => #{legacy_template}" do
173
+ let(:base_options) { super().merge(:client_settings => {:legacy_template => legacy_template}) }
174
+ subject { described_class.new(base_options) }
175
+ endpoint = legacy_template ? "_template" : "_index_template"
176
+ it "should call template_endpoint #{endpoint}" do
177
+ expect(subject.pool).to receive(:head).with("/#{endpoint}/#{template_name}").and_return(get_response)
178
+ expect(subject.pool).to receive(:put).with("/#{endpoint}/#{template_name}", nil, anything).and_return(get_response)
179
+ subject.template_install(template_name, template)
180
+ end
181
+ end
182
+ end
183
+ end
184
+
165
185
  describe "join_bulk_responses" do
166
186
  subject { described_class.new(base_options) }
167
187
 
@@ -13,27 +13,15 @@ require "logstash/outputs/opensearch/template_manager"
13
13
  describe LogStash::Outputs::OpenSearch::TemplateManager do
14
14
 
15
15
  describe ".default_template_path" do
16
- [1, 2].each do |major_version|
17
- context "when ECS is disabled with OpenSearch #{major_version}.x" do
18
- it 'resolves' do
19
- expect(described_class.default_template_path(major_version)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
20
- end
21
- it 'resolves' do
22
- expect(described_class.default_template_path(major_version, :disabled)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
23
- end
24
- end
25
- end
26
16
  [7, 1, 2].each do |major_version|
27
- context "when ECS v1 is requested with OpenSearch #{major_version}.x" do
28
- it 'resolves' do
29
- expect(described_class.default_template_path(major_version, :v1)).to end_with("/templates/ecs-v1/#{major_version}x.json")
30
- end
31
- end
32
- end
33
- [1, 2].each do |major_version|
34
- context "when ECS v8 is requested with OpenSearch #{major_version}.x" do
35
- it 'resolves' do
36
- expect(described_class.default_template_path(major_version, :v8)).to end_with("/templates/ecs-v8/#{major_version}x.json")
17
+ [:disabled, :v1, :v8].each do |ecs_ver|
18
+ [true, false].each do |legacy_template|
19
+ context "when ECS is #{ecs_ver} with OpenSearch #{major_version}.x legacy_template:#{legacy_template}" do
20
+ suffix = legacy_template ? "" : "_index"
21
+ it 'resolves' do
22
+ expect(described_class.default_template_path(major_version, ecs_ver, legacy_template)).to end_with("/templates/ecs-#{ecs_ver}/#{major_version}x#{suffix}.json")
23
+ end
24
+ end
37
25
  end
38
26
  end
39
27
  end
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-opensearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 2.0.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
@@ -31,7 +31,7 @@ cert_chain:
31
31
  r+j7FLyKuk5DzIxiCp8QN5dU71BbGUmsHf/C5UV76WLPOFX/szeaHhPwpjR3sK7r
32
32
  5zLgCV1KP7cgDdCYMlmZGeSViU8NV+Yy8/ghrzGpqVw=
33
33
  -----END CERTIFICATE-----
34
- date: 2022-08-17 00:00:00.000000000 Z
34
+ date: 2022-09-06 00:00:00.000000000 Z
35
35
  dependencies:
36
36
  - !ruby/object:Gem::Dependency
37
37
  requirement: !ruby/object:Gem::Requirement
@@ -56,23 +56,23 @@ dependencies:
56
56
  - !ruby/object:Gem::Dependency
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ">="
60
- - !ruby/object:Gem::Version
61
- version: 0.0.17
62
59
  - - "~>"
63
60
  - !ruby/object:Gem::Version
64
61
  version: '0.0'
62
+ - - ">="
63
+ - !ruby/object:Gem::Version
64
+ version: 0.0.17
65
65
  name: stud
66
66
  prerelease: false
67
67
  type: :runtime
68
68
  version_requirements: !ruby/object:Gem::Requirement
69
69
  requirements:
70
- - - ">="
71
- - !ruby/object:Gem::Version
72
- version: 0.0.17
73
70
  - - "~>"
74
71
  - !ruby/object:Gem::Version
75
72
  version: '0.0'
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: 0.0.17
76
76
  - !ruby/object:Gem::Dependency
77
77
  requirement: !ruby/object:Gem::Requirement
78
78
  requirements:
@@ -110,43 +110,37 @@ dependencies:
110
110
  - !ruby/object:Gem::Dependency
111
111
  requirement: !ruby/object:Gem::Requirement
112
112
  requirements:
113
- - - ">="
114
- - !ruby/object:Gem::Version
115
- version: 2.11.632
116
113
  - - "~>"
117
114
  - !ruby/object:Gem::Version
118
- version: '2'
115
+ version: '3'
119
116
  name: aws-sdk
120
117
  prerelease: false
121
118
  type: :runtime
122
119
  version_requirements: !ruby/object:Gem::Requirement
123
120
  requirements:
124
- - - ">="
125
- - !ruby/object:Gem::Version
126
- version: 2.11.632
127
121
  - - "~>"
128
122
  - !ruby/object:Gem::Version
129
- version: '2'
123
+ version: '3'
130
124
  - !ruby/object:Gem::Dependency
131
125
  requirement: !ruby/object:Gem::Requirement
132
126
  requirements:
133
- - - ">="
134
- - !ruby/object:Gem::Version
135
- version: 2.3.0
136
127
  - - "~>"
137
128
  - !ruby/object:Gem::Version
138
129
  version: '2'
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: 2.3.0
139
133
  name: json
140
134
  prerelease: false
141
135
  type: :runtime
142
136
  version_requirements: !ruby/object:Gem::Requirement
143
137
  requirements:
144
- - - ">="
145
- - !ruby/object:Gem::Version
146
- version: 2.3.0
147
138
  - - "~>"
148
139
  - !ruby/object:Gem::Version
149
140
  version: '2'
141
+ - - ">="
142
+ - !ruby/object:Gem::Version
143
+ version: 2.3.0
150
144
  - !ruby/object:Gem::Dependency
151
145
  requirement: !ruby/object:Gem::Requirement
152
146
  requirements:
@@ -237,6 +231,7 @@ files:
237
231
  - README.md
238
232
  - RELEASING.md
239
233
  - SECURITY.md
234
+ - docs/ecs_compatibility.md
240
235
  - lib/logstash/outputs/opensearch.rb
241
236
  - lib/logstash/outputs/opensearch/http_client.rb
242
237
  - lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb
@@ -244,11 +239,23 @@ files:
244
239
  - lib/logstash/outputs/opensearch/http_client_builder.rb
245
240
  - lib/logstash/outputs/opensearch/template_manager.rb
246
241
  - lib/logstash/outputs/opensearch/templates/ecs-disabled/1x.json
242
+ - lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json
247
243
  - lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json
244
+ - lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json
248
245
  - lib/logstash/outputs/opensearch/templates/ecs-disabled/7x.json
246
+ - lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json
247
+ - lib/logstash/outputs/opensearch/templates/ecs-v1/1x.json
248
+ - lib/logstash/outputs/opensearch/templates/ecs-v1/1x_index.json
249
+ - lib/logstash/outputs/opensearch/templates/ecs-v1/2x.json
250
+ - lib/logstash/outputs/opensearch/templates/ecs-v1/2x_index.json
251
+ - lib/logstash/outputs/opensearch/templates/ecs-v1/7x.json
252
+ - lib/logstash/outputs/opensearch/templates/ecs-v1/7x_index.json
249
253
  - lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json
254
+ - lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json
250
255
  - lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json
256
+ - lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json
251
257
  - lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json
258
+ - lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json
252
259
  - lib/logstash/plugin_mixins/opensearch/api_configs.rb
253
260
  - lib/logstash/plugin_mixins/opensearch/common.rb
254
261
  - logstash-output-opensearch.gemspec
@@ -306,7 +313,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
306
313
  - !ruby/object:Gem::Version
307
314
  version: '0'
308
315
  requirements: []
309
- rubygems_version: 3.3.20
316
+ rubyforge_project:
317
+ rubygems_version: 2.7.10
310
318
  signing_key:
311
319
  specification_version: 4
312
320
  summary: Stores logs in OpenSearch
metadata.gz.sig CHANGED
Binary file