logstash-output-elasticsearch 10.8.6-java → 11.0.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (31) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +7 -0
  3. data/docs/index.asciidoc +132 -22
  4. data/lib/logstash/outputs/elasticsearch.rb +122 -64
  5. data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +233 -0
  6. data/lib/logstash/outputs/elasticsearch/http_client.rb +9 -7
  7. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +47 -34
  8. data/lib/logstash/outputs/elasticsearch/ilm.rb +11 -12
  9. data/lib/logstash/outputs/elasticsearch/license_checker.rb +19 -22
  10. data/lib/logstash/outputs/elasticsearch/template_manager.rb +3 -5
  11. data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +157 -153
  12. data/lib/logstash/plugin_mixins/elasticsearch/common.rb +70 -58
  13. data/logstash-output-elasticsearch.gemspec +2 -2
  14. data/spec/es_spec_helper.rb +3 -6
  15. data/spec/integration/outputs/data_stream_spec.rb +61 -0
  16. data/spec/integration/outputs/ilm_spec.rb +6 -2
  17. data/spec/integration/outputs/ingest_pipeline_spec.rb +4 -2
  18. data/spec/integration/outputs/retry_spec.rb +4 -4
  19. data/spec/integration/outputs/sniffer_spec.rb +0 -1
  20. data/spec/spec_helper.rb +14 -0
  21. data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +542 -0
  22. data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +1 -0
  23. data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +24 -10
  24. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +2 -3
  25. data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +1 -3
  26. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +1 -2
  27. data/spec/unit/outputs/elasticsearch_spec.rb +122 -23
  28. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +1 -2
  29. data/spec/unit/outputs/error_whitelist_spec.rb +3 -2
  30. data/spec/unit/outputs/license_check_spec.rb +0 -16
  31. metadata +23 -16
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '10.8.6'
3
+ s.version = '11.0.0'
4
4
 
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Stores logs in Elasticsearch"
@@ -23,13 +23,13 @@ Gem::Specification.new do |s|
23
23
 
24
24
  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
25
25
  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
26
- s.add_runtime_dependency 'cabin', ['~> 0.6']
27
26
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
28
27
  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
29
28
 
30
29
  s.add_development_dependency 'logstash-codec-plain'
31
30
  s.add_development_dependency 'logstash-devutils'
32
31
  s.add_development_dependency 'flores'
32
+ s.add_development_dependency 'cabin', ['~> 0.6']
33
33
  # Still used in some specs, we should remove this ASAP
34
34
  s.add_development_dependency 'elasticsearch'
35
35
  end
@@ -1,5 +1,5 @@
1
- require "logstash/devutils/rspec/spec_helper"
2
- require 'manticore'
1
+ require_relative './spec_helper'
2
+
3
3
  require 'elasticsearch'
4
4
  require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
5
5
  require_relative "support/elasticsearch/api/actions/get_alias"
@@ -8,10 +8,7 @@ require_relative "support/elasticsearch/api/actions/get_ilm_policy"
8
8
  require_relative "support/elasticsearch/api/actions/put_ilm_policy"
9
9
 
10
10
  require 'json'
11
-
12
- unless defined?(LogStash::OSS)
13
- LogStash::OSS = ENV['DISTRIBUTION'] != "default"
14
- end
11
+ require 'cabin'
15
12
 
16
13
  module ESHelper
17
14
  def get_host_port
@@ -0,0 +1,61 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+ require "logstash/outputs/elasticsearch"
3
+
4
+ describe "data streams", :integration => true do
5
+
6
+ let(:ds_name) { "logs-#{ds_dataset}-default" }
7
+ let(:ds_dataset) { 'integration_test' }
8
+
9
+ let(:options) do
10
+ { "data_stream" => 'true', "data_stream_dataset" => ds_dataset, "hosts" => get_host_port() }
11
+ end
12
+
13
+ subject { LogStash::Outputs::ElasticSearch.new(options) }
14
+
15
+ before :each do
16
+ @es = get_client
17
+ @es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil
18
+
19
+ es_version = @es.info['version']['number']
20
+ if Gem::Version.create(es_version) < Gem::Version.create('7.9.0')
21
+ skip "ES version #{es_version} does not support data-streams"
22
+ end
23
+ end
24
+
25
+ it "creates a new document" do
26
+ subject.register
27
+ subject.multi_receive([LogStash::Event.new("message" => "MSG 111")])
28
+
29
+ @es.indices.refresh
30
+
31
+ Stud::try(3.times) do
32
+ r = @es.search(index: ds_name)
33
+
34
+ expect( r['hits']['total']['value'] ).to eq 1
35
+ doc = r['hits']['hits'].first
36
+ expect( doc['_source'] ).to include "message"=>"MSG 111"
37
+ expect( doc['_source'] ).to include "data_stream"=>{"dataset"=>ds_dataset, "type"=>"logs", "namespace"=>"default"}
38
+ end
39
+ end
40
+
41
+ context "with document_id" do
42
+
43
+ let(:document_id) { '1234567890' }
44
+ let(:options) { super().merge("document_id" => document_id) }
45
+
46
+ it "creates a new document" do
47
+ subject.register
48
+ subject.multi_receive([LogStash::Event.new("message" => "foo")])
49
+
50
+ @es.indices.refresh
51
+
52
+ Stud::try(3.times) do
53
+ r = @es.search(index: ds_name, body: { query: { match: { _id: document_id } } })
54
+ expect( r['hits']['total']['value'] ).to eq 1
55
+ doc = r['hits']['hits'].first
56
+ expect( doc['_source'] ).to include "message"=>"foo"
57
+ end
58
+ end
59
+
60
+ end
61
+ end
@@ -198,12 +198,16 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
198
198
  let (:settings) { super().merge!({ 'ilm_enabled' => true }) }
199
199
 
200
200
  it 'should raise a configuration error' do
201
+ # TODO should be refactored not to rely on plugin internals
202
+ finish_register = subject.method(:finish_register)
203
+ expect(subject).to receive(:finish_register)
201
204
  expect do
202
205
  begin
203
206
  subject.register
204
- sleep(1)
207
+ finish_register.call
208
+ sleep(1.5) # wait_for_successful_connection (for the thread to raise)
205
209
  ensure
206
- subject.stop_template_installer
210
+ subject.send :stop_after_successful_connection_thread
207
211
  end
208
212
  end.to raise_error(LogStash::ConfigurationError)
209
213
  end
@@ -6,7 +6,8 @@ if ESHelper.es_version_satisfies?(">= 5")
6
6
  require "logstash/outputs/elasticsearch"
7
7
  settings = {
8
8
  "hosts" => "#{get_host_port()}",
9
- "pipeline" => "apache-logs"
9
+ "pipeline" => "apache-logs",
10
+ "data_stream" => 'false'
10
11
  }
11
12
  next LogStash::Outputs::ElasticSearch.new(settings)
12
13
  end
@@ -52,9 +53,10 @@ if ESHelper.es_version_satisfies?(">= 5")
52
53
  @es.indices.refresh
53
54
 
54
55
  #Wait or fail until everything's indexed.
55
- Stud::try(20.times) do
56
+ Stud::try(10.times) do
56
57
  r = @es.search(index: 'logstash-*')
57
58
  expect(r).to have_hits(1)
59
+ sleep(0.1)
58
60
  end
59
61
  end
60
62
 
@@ -6,17 +6,17 @@ describe "failures in bulk class expected behavior", :integration => true do
6
6
  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
7
7
  let(:action1) do
8
8
  if ESHelper.es_version_satisfies?(">= 6", "< 7")
9
- ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1])
9
+ ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1.to_hash])
10
10
  else
11
- ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event1])
11
+ ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event1.to_hash])
12
12
  end
13
13
  end
14
14
  let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
15
15
  let(:action2) do
16
16
  if ESHelper.es_version_satisfies?(">= 6", "< 7")
17
- ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2])
17
+ ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2.to_hash])
18
18
  else
19
- ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event2])
19
+ ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event2.to_hash])
20
20
  end
21
21
  end
22
22
  let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
@@ -1,4 +1,3 @@
1
- require "logstash/devutils/rspec/spec_helper"
2
1
  require_relative "../../../spec/es_spec_helper"
3
2
  require "logstash/outputs/elasticsearch/http_client"
4
3
  require "json"
@@ -0,0 +1,14 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+
3
+ unless defined?(LogStash::OSS)
4
+ LogStash::OSS = ENV['DISTRIBUTION'] != "default"
5
+ end
6
+
7
+ require "logstash/outputs/elasticsearch"
8
+
9
+ module LogStash::Outputs::ElasticSearch::SpecHelper
10
+ end
11
+
12
+ RSpec.configure do |config|
13
+ config.include LogStash::Outputs::ElasticSearch::SpecHelper
14
+ end
@@ -0,0 +1,542 @@
1
+ require_relative '../../../../spec/spec_helper'
2
+ require "logstash/outputs/elasticsearch/data_stream_support"
3
+
4
+ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
5
+
6
+ subject { LogStash::Outputs::ElasticSearch.new(options) }
7
+ let(:options) { { 'hosts' => [ 'localhost:12345' ] } }
8
+ let(:es_version) { '7.10.1' }
9
+
10
+ let(:do_register) { false }
11
+ let(:stub_plugin_register!) do
12
+ allow(subject).to receive(:last_es_version).and_return(es_version)
13
+
14
+ allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:start)
15
+
16
+ # stub-out unrelated (finish_register) setup:
17
+ allow(subject).to receive(:discover_cluster_uuid)
18
+ allow(subject).to receive(:install_template)
19
+ allow(subject).to receive(:ilm_in_use?).and_return nil
20
+
21
+ # emulate 'successful' ES connection on the same thread
22
+ allow(subject).to receive(:after_successful_connection) { |&block| block.call }
23
+ allow(subject).to receive(:stop_after_successful_connection_thread)
24
+
25
+ subject.register
26
+
27
+ allow(subject.client).to receive(:maximum_seen_major_version).and_return(Integer(es_version.split('.').first))
28
+
29
+ # allow( subject.logger ).to receive(:info) do |msg|
30
+ # expect(msg).to include "New Elasticsearch output"
31
+ # end
32
+ end
33
+
34
+ @@logstash_oss = LogStash::OSS
35
+
36
+ before(:each) do
37
+ change_constant :OSS, false, target: LogStash # assume non-OSS by default
38
+
39
+ stub_plugin_register! if do_register
40
+ end
41
+
42
+ after(:each) do
43
+ subject.close if do_register
44
+
45
+ change_constant :OSS, @@logstash_oss, target: LogStash
46
+ end
47
+
48
+ context "default configuration" do
49
+
50
+ let(:options) { {} }
51
+
52
+ before { allow(subject).to receive(:last_es_version).and_return(es_version) }
53
+
54
+ it "does not use data-streams on LS 7.x" do
55
+ change_constant :LOGSTASH_VERSION, '7.10.0' do
56
+ expect( subject.data_stream_config? ).to be false
57
+ end
58
+ end
59
+
60
+ it "warns when configuration is data-stream compliant (LS 7.x)" do
61
+ expect( subject.logger ).to receive(:warn) do |msg|
62
+ expect(msg).to include "Configuration is data stream compliant but due backwards compatibility Logstash 7.x"
63
+ end
64
+ change_constant :LOGSTASH_VERSION, '7.11.0' do
65
+ expect( subject.data_stream_config? ).to be false
66
+ end
67
+ end
68
+
69
+ it "defaults to using data-streams on LS 8.0" do
70
+ change_constant :LOGSTASH_VERSION, '8.0.0' do
71
+ expect( subject.data_stream_config? ).to be true
72
+ end
73
+ end
74
+
75
+ context 'non-compatible ES' do
76
+
77
+ let(:es_version) { '7.8.0' }
78
+
79
+ it "does not print an error (from after_successful_connection thread)" do
80
+ change_constant :LOGSTASH_VERSION, '7.8.1' do
81
+ expect( subject.logger ).to_not receive(:error)
82
+ expect( subject ).to receive(:finish_register).once.and_call_original
83
+ stub_plugin_register!
84
+ end
85
+ end
86
+
87
+ end
88
+
89
+ end
90
+
91
+ context "ds-compatible configuration" do
92
+
93
+ let(:options) do
94
+ {
95
+ 'hosts' => [ 'http://127.0.0.1:12345' ],
96
+ 'http_compression' => 'true', 'bulk_path' => '_bulk', 'timeout' => '30',
97
+ 'user' => 'elastic', 'password' => 'ForSearch!', 'ssl' => 'false'
98
+ }
99
+ end
100
+
101
+ before { allow(subject).to receive(:last_es_version).and_return(es_version) }
102
+
103
+ it "does not use data-streams on LS 7.x" do
104
+ change_constant :LOGSTASH_VERSION, '7.10.0' do
105
+ expect( subject.data_stream_config? ).to be false
106
+ end
107
+ end
108
+
109
+ it "defaults to using data-streams on LS 8.0" do
110
+ change_constant :LOGSTASH_VERSION, '8.0.1' do
111
+ expect( subject.data_stream_config? ).to be true
112
+ end
113
+ change_constant :LOGSTASH_VERSION, '8.1.0' do
114
+ expect( subject.send(:check_data_stream_config!) ).to be true
115
+ end
116
+ end
117
+
118
+ it "uses data-streams on LS 8.0 (OSS)" do
119
+ change_constant :LOGSTASH_VERSION, '8.0.1' do
120
+ change_constant :OSS, true, target: LogStash do
121
+ expect( subject.data_stream_config? ).to be true
122
+ end
123
+ end
124
+ end
125
+
126
+ context 'old ES' do
127
+
128
+ let(:es_version) { '7.8.1' }
129
+
130
+ it "prints an error (from after_successful_connection thread) on LS 8.0" do
131
+ change_constant :LOGSTASH_VERSION, '8.0.0' do
132
+ expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
133
+ {:es_version=>"7.8.1"})
134
+ stub_plugin_register!
135
+ end
136
+ end
137
+
138
+ end
139
+
140
+ end
141
+
142
+ context "default (non data-stream) configuration (on 7.x)" do
143
+
144
+ let(:options) do
145
+ { 'data_stream_dataset' => 'test', 'data_stream_auto_routing' => 'false', 'user' => 'elastic' }
146
+ end
147
+
148
+ before { allow(subject).to receive(:last_es_version).and_return(es_version) }
149
+
150
+ it "does not default to data-streams" do
151
+ expect( subject.logger ).to receive(:error) do |msg|
152
+ expect(msg).to include "Ambiguous configuration; data stream settings are present, but data streams are not enabled"
153
+ end
154
+ change_constant :LOGSTASH_VERSION, '7.10.2' do
155
+ expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
156
+ end
157
+ end
158
+
159
+ context 'explicit data_stream => false' do
160
+
161
+ let(:options) { super().merge('data_stream' => 'false') }
162
+
163
+ it "raises a configuration error (due ds specific settings)" do
164
+ expect( subject.logger ).to receive(:error).with(/Ambiguous configuration; data stream settings must not be present when data streams is disabled/,
165
+ {"data_stream_auto_routing"=>"false", "data_stream_dataset"=>"test"})
166
+ change_constant :LOGSTASH_VERSION, '7.10.2' do
167
+ expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
168
+ end
169
+ end
170
+
171
+ end
172
+
173
+ end
174
+
175
+ context "(explicit) ds disabled configuration" do
176
+
177
+ let(:options) { super().merge('data_stream' => false.to_s) }
178
+
179
+ before { allow(subject).to receive(:last_es_version).and_return(es_version) }
180
+
181
+ it "does not use data-streams on LS 7.x" do
182
+ change_constant :LOGSTASH_VERSION, '7.10.0' do
183
+ expect( subject.data_stream_config? ).to be false
184
+ end
185
+ end
186
+
187
+ it "does not use data-streams on LS 8.0" do
188
+ change_constant :LOGSTASH_VERSION, '8.0.0' do
189
+ expect( subject.data_stream_config? ).to be false
190
+ end
191
+ end
192
+
193
+ it "does not print a warning" do
194
+ expect( subject.logger ).to_not receive(:warn)
195
+ change_constant :LOGSTASH_VERSION, '7.10.2' do
196
+ expect( subject.data_stream_config? ).to be false
197
+ end
198
+ end
199
+
200
+ end
201
+
202
+ context "(explicit) ds enabled configuration" do
203
+
204
+ let(:options) { super().merge('data_stream' => true.to_s) }
205
+
206
+ before { allow(subject).to receive(:last_es_version).and_return(es_version) }
207
+
208
+ it "does use data-streams on LS 7.x" do
209
+ change_constant :LOGSTASH_VERSION, '7.9.1' do
210
+ expect( subject.data_stream_config? ).to be true
211
+ end
212
+ end
213
+
214
+ it "does use data-streams on LS 8.0" do
215
+ change_constant :LOGSTASH_VERSION, '8.1.0' do
216
+ expect( subject.data_stream_config? ).to be true
217
+ end
218
+ end
219
+
220
+ context 'non-compatible ES' do
221
+
222
+ let(:es_version) { '6.8.11' }
223
+
224
+ it "prints an error (from after_successful_connection thread) on LS 7.x" do
225
+ change_constant :LOGSTASH_VERSION, '7.12.0' do
226
+ expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
227
+ {:es_version=>"6.8.11"})
228
+ stub_plugin_register!
229
+ end
230
+ end
231
+
232
+ it "prints an error (from after_successful_connection thread) on LS 8.0" do
233
+ change_constant :LOGSTASH_VERSION, '8.0.5' do
234
+ expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
235
+ {:es_version=>"6.8.11"})
236
+ stub_plugin_register!
237
+ end
238
+ end
239
+
240
+ end
241
+
242
+ end
243
+
244
+ describe "auto routing" do
245
+
246
+ let(:options) { super().merge('data_stream' => 'true') }
247
+ let(:do_register) { true }
248
+
249
+ let(:event) do
250
+ event = LogStash::Event.new
251
+ event.set '[host][hostname]', 'orangutan'
252
+ event
253
+ end
254
+
255
+ context 'with data_stream.* event data' do
256
+
257
+ let(:event) do
258
+ super().tap do |event|
259
+ event.set '[data_stream][type]', 'metrics'
260
+ event.set '[data_stream][dataset]', 'src1'
261
+ event.set '[data_stream][namespace]', 'test'
262
+ end
263
+ end
264
+
265
+ it 'uses event specified target' do
266
+ tuple = subject.map_events([ event ]).first
267
+ expect( tuple.size ).to eql 3
268
+ expect( tuple[0] ).to eql 'create'
269
+ expect( tuple[1] ).to include :_index => 'metrics-src1-test'
270
+ end
271
+
272
+ end
273
+
274
+ context 'with routing turned off' do
275
+
276
+ let(:options) { super().merge('data_stream_auto_routing' => 'false') }
277
+
278
+ let(:event) do
279
+ super().tap do |event|
280
+ event.set '[data_stream][type]', 'metrics'
281
+ event.set '[data_stream][dataset]', 'src1'
282
+ event.set '[data_stream][namespace]', 'test'
283
+ end
284
+ end
285
+
286
+ it 'uses event specified target' do
287
+ tuple = subject.map_events([ event ]).first
288
+ expect( tuple.size ).to eql 3
289
+ expect( tuple[0] ).to eql 'create'
290
+ expect( tuple[1] ).to include :_index => 'logs-generic-default'
291
+ end
292
+
293
+ end
294
+
295
+ context 'with partial data_stream.* data' do
296
+
297
+ let(:options) { super().merge('data_stream_dataset' => 'data') }
298
+
299
+ let(:event) do
300
+ super().tap do |event|
301
+ event.set '[data_stream][type]', 'metrics'
302
+ event.set '[data_stream][dataset]', 'src1'
303
+ end
304
+ end
305
+
306
+ it 'uses event specified target' do
307
+ tuple = subject.map_events([ event ]).first
308
+ expect( tuple.size ).to eql 3
309
+ expect( tuple[0] ).to eql 'create'
310
+ expect( tuple[1] ).to include :_index => 'metrics-src1-default'
311
+ end
312
+
313
+ end
314
+
315
+ context 'with no data_stream.* fields' do
316
+
317
+ let(:options) { super().merge('data_stream_dataset' => 'stats', 'data_stream_type' => 'metrics') }
318
+
319
+ it 'uses configuration target' do
320
+ tuple = subject.map_events([ event ]).first
321
+ expect( tuple.size ).to eql 3
322
+ expect( tuple[0] ).to eql 'create'
323
+ expect( tuple[1] ).to include :_index => 'metrics-stats-default'
324
+ end
325
+
326
+ end
327
+
328
+ context 'with default configuration' do
329
+
330
+ it 'uses default target' do
331
+ tuple = subject.map_events([ event ]).first
332
+ expect( tuple.size ).to eql 3
333
+ expect( tuple[0] ).to eql 'create'
334
+ expect( tuple[1] ).to include :_index => 'logs-generic-default'
335
+ end
336
+
337
+ end
338
+
339
+ end
340
+
341
+ describe "field sync" do
342
+
343
+ let(:options) { super().merge('data_stream' => 'true') }
344
+
345
+ let(:do_register) { true }
346
+
347
+ let(:event) do
348
+ event = LogStash::Event.new
349
+ event.set '[host][hostname]', 'orangutan'
350
+ event
351
+ end
352
+
353
+ context 'enabled and no event data' do
354
+
355
+ let(:options) { super().merge('data_stream_sync_fields' => 'true') }
356
+
357
+ it 'fills in DS fields' do
358
+ tuple = subject.map_events([ event ]).first
359
+ expect( tuple.size ).to eql 3
360
+ expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "default"})
361
+ end
362
+
363
+ end
364
+
365
+ context 'enabled and some event data' do
366
+
367
+ let(:options) { super().merge('data_stream_dataset' => 'ds1', 'data_stream_sync_fields' => 'true') }
368
+
369
+ let(:event) do
370
+ super().tap do |event|
371
+ event.set '[data_stream][namespace]', 'custom'
372
+ end
373
+ end
374
+
375
+ it 'fills in missing fields' do
376
+ tuple = subject.map_events([ event ]).first
377
+ expect( tuple.size ).to eql 3
378
+ expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "ds1", "namespace" => "custom"})
379
+ end
380
+
381
+ it 'does not mutate data_stream hash' do
382
+ data_stream = event.get('data_stream')
383
+ data_stream_dup = data_stream.dup
384
+ subject.map_events([ event ])
385
+ expect( data_stream ).to eql data_stream_dup
386
+ end
387
+
388
+ end
389
+
390
+ context 'enabled with invalid data' do
391
+
392
+ let(:options) { super().merge('data_stream_sync_fields' => 'true') }
393
+
394
+ let(:event) do
395
+ super().tap do |event|
396
+ event.set '[data_stream]', false
397
+ end
398
+ end
399
+
400
+ it 'overwrites invalid data_stream field' do
401
+ tuple = subject.map_events([ event ]).first
402
+ expect( tuple.size ).to eql 3
403
+ expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "default"})
404
+ end
405
+
406
+ end
407
+
408
+ context 'enabled having invalid data with routing disabled' do
409
+
410
+ let(:options) do
411
+ super().merge('data_stream_sync_fields' => 'true', 'data_stream_auto_routing' => 'false', 'data_stream_namespace' => 'ns1')
412
+ end
413
+
414
+ let(:event) do
415
+ super().tap do |event|
416
+ event.set '[data_stream][type]', 'foo'
417
+ event.set '[data_stream][dataset]', false
418
+ event.set '[data_stream][extra]', 0
419
+ end
420
+ end
421
+
422
+ it 'overwrites invalid data_stream sub-fields' do
423
+ tuple = subject.map_events([ event ]).first
424
+ expect( tuple.size ).to eql 3
425
+ expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "ns1", "extra" => 0})
426
+ end
427
+
428
+ end
429
+
430
+ context 'disabled and no event data' do
431
+
432
+ let(:options) { super().merge('data_stream_dataset' => 'ds1', 'data_stream_sync_fields' => 'false') }
433
+
434
+ it 'does not fill DS fields' do
435
+ tuple = subject.map_events([ event ]).first
436
+ expect( tuple.size ).to eql 3
437
+ expect( tuple[2].keys ).to_not include 'data_stream'
438
+ end
439
+
440
+ end
441
+
442
+ context 'disabled and some event data' do
443
+
444
+ let(:options) { super().merge('data_stream_sync_fields' => 'false') }
445
+
446
+ let(:event) do
447
+ super().tap do |event|
448
+ event.set '[data_stream][type]', 'logs'
449
+ end
450
+ end
451
+
452
+ it 'does not fill DS fields' do
453
+ tuple = subject.map_events([ event ]).first
454
+ expect( tuple.size ).to eql 3
455
+ expect( tuple[2]['data_stream'] ).to eql({ 'type' => 'logs'})
456
+ end
457
+
458
+ end
459
+
460
+ end
461
+
462
+ describe "validation" do
463
+
464
+ context 'with too long dataset name' do
465
+
466
+ let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => 'x' * 120) }
467
+
468
+ it 'fails' do
469
+ expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
470
+ end
471
+
472
+ end
473
+
474
+ context 'with empty dataset name' do
475
+
476
+ let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => '') }
477
+
478
+ it 'fails' do
479
+ expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
480
+ end
481
+
482
+ end
483
+
484
+ context 'with invalid dataset char' do
485
+
486
+ let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => 'foo/bar') }
487
+
488
+ it 'fails' do
489
+ expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
490
+ end
491
+
492
+ end
493
+
494
+ context 'with invalid namespace char' do
495
+
496
+ let(:options) { super().merge('data_stream' => 'true', 'data_stream_namespace' => 'foo*') }
497
+
498
+ it 'fails' do
499
+ expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
500
+ end
501
+
502
+ end
503
+
504
+ context 'with invalid "empty" namespace' do
505
+
506
+ let(:options) { super().merge('data_stream' => 'true', 'data_stream_namespace' => ' ') }
507
+
508
+ it 'fails' do
509
+ expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
510
+ end
511
+
512
+ end
513
+
514
+ context 'with invalid type' do
515
+
516
+ let(:options) { super().merge('data_stream' => 'true', 'data_stream_type' => 'custom') }
517
+
518
+ it 'fails' do
519
+ expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
520
+ end
521
+
522
+ end
523
+
524
+ end
525
+
526
+ private
527
+
528
+ def change_constant(name, new_value, target: Object)
529
+ old_value = target.const_get name
530
+ begin
531
+ target.send :remove_const, name
532
+ target.const_set name, new_value
533
+ yield if block_given?
534
+ ensure
535
+ if block_given?
536
+ target.send :remove_const, name rescue nil
537
+ target.const_set name, old_value
538
+ end
539
+ end
540
+ end
541
+
542
+ end