logstash-output-amazon_es 2.0.1-java → 6.4.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CONTRIBUTORS +12 -0
- data/Gemfile +8 -0
- data/LICENSE +10 -199
- data/README.md +34 -65
- data/lib/logstash/outputs/amazon_es.rb +218 -423
- data/lib/logstash/outputs/amazon_es/common.rb +347 -0
- data/lib/logstash/outputs/amazon_es/common_configs.rb +141 -0
- data/lib/logstash/outputs/amazon_es/elasticsearch-template-es2x.json +95 -0
- data/lib/logstash/outputs/amazon_es/elasticsearch-template-es5x.json +46 -0
- data/lib/logstash/outputs/amazon_es/elasticsearch-template-es6x.json +45 -0
- data/lib/logstash/outputs/amazon_es/elasticsearch-template-es7x.json +46 -0
- data/lib/logstash/outputs/amazon_es/http_client.rb +359 -74
- data/lib/logstash/outputs/amazon_es/http_client/manticore_adapter.rb +169 -0
- data/lib/logstash/outputs/amazon_es/http_client/pool.rb +457 -0
- data/lib/logstash/outputs/amazon_es/http_client_builder.rb +164 -0
- data/lib/logstash/outputs/amazon_es/template_manager.rb +36 -0
- data/logstash-output-amazon_es.gemspec +13 -22
- data/spec/es_spec_helper.rb +37 -0
- data/spec/unit/http_client_builder_spec.rb +189 -0
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +105 -0
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +198 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +222 -0
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +25 -0
- data/spec/unit/outputs/elasticsearch_spec.rb +615 -0
- data/spec/unit/outputs/error_whitelist_spec.rb +60 -0
- metadata +49 -110
- data/lib/logstash/outputs/amazon_es/aws_transport.rb +0 -109
- data/lib/logstash/outputs/amazon_es/aws_v4_signer.rb +0 -7
- data/lib/logstash/outputs/amazon_es/aws_v4_signer_impl.rb +0 -62
- data/lib/logstash/outputs/amazon_es/elasticsearch-template.json +0 -41
- data/spec/amazon_es_spec_helper.rb +0 -69
- data/spec/unit/outputs/amazon_es_spec.rb +0 -50
- data/spec/unit/outputs/elasticsearch/protocol_spec.rb +0 -36
- data/spec/unit/outputs/elasticsearch_proxy_spec.rb +0 -58
@@ -0,0 +1,25 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/amazon_es/http_client"
|
3
|
+
require "java"
|
4
|
+
require "json"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::ElasticSearch::TemplateManager do
|
7
|
+
|
8
|
+
describe ".default_template_path" do
|
9
|
+
context "amazon_es 1.x" do
|
10
|
+
it "chooses the 2x template" do
|
11
|
+
expect(described_class.default_template_path(1)).to match(/elasticsearch-template-es2x.json/)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
context "amazon_es 2.x" do
|
15
|
+
it "chooses the 2x template" do
|
16
|
+
expect(described_class.default_template_path(2)).to match(/elasticsearch-template-es2x.json/)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
context "amazon_es 5.x" do
|
20
|
+
it "chooses the 5x template" do
|
21
|
+
expect(described_class.default_template_path(5)).to match(/elasticsearch-template-es5x.json/)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,615 @@
|
|
1
|
+
require_relative "../../../spec/es_spec_helper"
|
2
|
+
require "flores/random"
|
3
|
+
require "logstash/outputs/amazon_es"
|
4
|
+
|
5
|
+
describe LogStash::Outputs::ElasticSearch do
|
6
|
+
subject { described_class.new(options) }
|
7
|
+
let(:options) { { "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
8
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
9
|
+
let(:maximum_seen_major_version) { rand(100) }
|
10
|
+
|
11
|
+
let(:do_register) { true }
|
12
|
+
|
13
|
+
before(:each) do
|
14
|
+
if do_register
|
15
|
+
subject.register
|
16
|
+
|
17
|
+
# Rspec mocks can't handle background threads, so... we can't use any
|
18
|
+
allow(subject.client.pool).to receive(:start_resurrectionist)
|
19
|
+
allow(subject.client.pool).to receive(:start_sniffer)
|
20
|
+
allow(subject.client.pool).to receive(:healthcheck!)
|
21
|
+
allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
|
22
|
+
subject.client.pool.adapter.manticore.respond_with(:body => "{}")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
after(:each) do
|
27
|
+
subject.close
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
context "with an active instance" do
|
32
|
+
let(:options) {
|
33
|
+
{
|
34
|
+
"index" => "my-index",
|
35
|
+
"hosts" => ["localhost"],
|
36
|
+
"path" => "some-path",
|
37
|
+
"manage_template" => false,
|
38
|
+
"port" => 9200,
|
39
|
+
"protocol" => "http",
|
40
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
41
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
42
|
+
}
|
43
|
+
}
|
44
|
+
|
45
|
+
let(:manticore_urls) { subject.client.pool.urls }
|
46
|
+
let(:manticore_url) { manticore_urls.first }
|
47
|
+
|
48
|
+
describe "getting a document type" do
|
49
|
+
context "if document_type isn't set" do
|
50
|
+
let(:options) { super.merge("document_type" => nil)}
|
51
|
+
context "for 7.x amazon_es clusters" do
|
52
|
+
let(:maximum_seen_major_version) { 7 }
|
53
|
+
it "should return '_doc'" do
|
54
|
+
expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("_doc")
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
context "for 6.x amazon_es clusters" do
|
59
|
+
let(:maximum_seen_major_version) { 6 }
|
60
|
+
it "should return 'doc'" do
|
61
|
+
expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
context "for < 6.0 amazon_es clusters" do
|
66
|
+
let(:maximum_seen_major_version) { 5 }
|
67
|
+
it "should get the type from the event" do
|
68
|
+
expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
context "with 'document type set'" do
|
74
|
+
let(:options) { super.merge("document_type" => "bar")}
|
75
|
+
it "should get the event type from the 'document_type' setting" do
|
76
|
+
expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar")
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
context "with a bad type event field in a < 6.0 es cluster" do
|
81
|
+
let(:maximum_seen_major_version) { 5 }
|
82
|
+
let(:type_arg) { ["foo"] }
|
83
|
+
let(:result) { subject.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
|
84
|
+
|
85
|
+
before do
|
86
|
+
allow(subject.instance_variable_get(:@logger)).to receive(:warn)
|
87
|
+
result
|
88
|
+
end
|
89
|
+
|
90
|
+
it "should call @logger.warn and return nil" do
|
91
|
+
expect(subject.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
|
92
|
+
end
|
93
|
+
|
94
|
+
it "should set the type to the stringified value" do
|
95
|
+
expect(result).to eql(type_arg.to_s)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
describe "with auth" do
|
101
|
+
let(:user) { "myuser" }
|
102
|
+
let(:password) { ::LogStash::Util::Password.new("mypassword") }
|
103
|
+
|
104
|
+
shared_examples "an authenticated config" do
|
105
|
+
it "should set the URL auth correctly" do
|
106
|
+
expect(manticore_url.user).to eq user
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
context "as part of a URL" do
|
111
|
+
let(:options) {
|
112
|
+
super.merge("hosts" => ["http://#{user}:#{password.value}@localhost:9200"])
|
113
|
+
}
|
114
|
+
|
115
|
+
include_examples("an authenticated config")
|
116
|
+
end
|
117
|
+
|
118
|
+
context "as a hash option" do
|
119
|
+
let(:options) {
|
120
|
+
super.merge!(
|
121
|
+
"user" => user,
|
122
|
+
"password" => password
|
123
|
+
)
|
124
|
+
}
|
125
|
+
|
126
|
+
include_examples("an authenticated config")
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
describe "with path" do
|
131
|
+
it "should properly create a URI with the path" do
|
132
|
+
expect(subject.path).to eql(options["path"])
|
133
|
+
end
|
134
|
+
|
135
|
+
it "should properly set the path on the HTTP client adding slashes" do
|
136
|
+
expect(manticore_url.path).to eql("/" + options["path"] + "/")
|
137
|
+
end
|
138
|
+
|
139
|
+
context "with extra slashes" do
|
140
|
+
let(:path) { "/slashed-path/ "}
|
141
|
+
let(:options) { super.merge("path" => "/some-path/") }
|
142
|
+
|
143
|
+
it "should properly set the path on the HTTP client without adding slashes" do
|
144
|
+
expect(manticore_url.path).to eql(options["path"])
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
context "with a URI based path" do
|
149
|
+
let(:options) do
|
150
|
+
o = super()
|
151
|
+
o.delete("pat˙h")
|
152
|
+
o["hosts"] = ["http://localhost:9200/mypath/"]
|
153
|
+
o["port"] = 9200
|
154
|
+
o["protocol"] = 'http'
|
155
|
+
o["aws_access_key_id"] = "AAAAAAAAAAAAAAAAAAAA"
|
156
|
+
o["aws_secret_access_key"] = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
157
|
+
o
|
158
|
+
end
|
159
|
+
let(:client_host_path) { manticore_url.path }
|
160
|
+
|
161
|
+
|
162
|
+
context "with a path option but no URL path" do
|
163
|
+
let(:options) do
|
164
|
+
o = super()
|
165
|
+
o["path"] = "/override/"
|
166
|
+
o["hosts"] = ["http://localhost:9200"]
|
167
|
+
o["port"] = 9200
|
168
|
+
o["protocol"] = 'http'
|
169
|
+
o["aws_access_key_id"] = "AAAAAAAAAAAAAAAAAAAA"
|
170
|
+
o["aws_secret_access_key"] = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
171
|
+
o
|
172
|
+
end
|
173
|
+
|
174
|
+
it "should initialize without error" do
|
175
|
+
expect { subject }.not_to raise_error
|
176
|
+
end
|
177
|
+
|
178
|
+
it "should use the option path" do
|
179
|
+
expect(client_host_path).to eql("/override/")
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
# If you specify the path in two spots that is an error!
|
184
|
+
context "with a path option and a URL path" do
|
185
|
+
let(:do_register) { false } # Register will fail
|
186
|
+
let(:options) do
|
187
|
+
o = super()
|
188
|
+
o["path"] = "/override"
|
189
|
+
o["hosts"] = ["http://localhost:9200/mypath/"]
|
190
|
+
o
|
191
|
+
end
|
192
|
+
|
193
|
+
it "should initialize with an error" do
|
194
|
+
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
describe "without a port specified" do
|
201
|
+
let(:options) { super.merge('hosts' => 'localhost') }
|
202
|
+
it "should properly set the default port (9200) on the HTTP client" do
|
203
|
+
expect(manticore_url.port).to eql(9200)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
describe "#multi_receive" do
|
208
|
+
let(:events) { [double("one"), double("two"), double("three")] }
|
209
|
+
let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
|
210
|
+
|
211
|
+
before do
|
212
|
+
allow(subject).to receive(:retrying_submit).with(anything)
|
213
|
+
events.each_with_index do |e,i|
|
214
|
+
et = events_tuples[i]
|
215
|
+
allow(subject).to receive(:event_action_tuple).with(e).and_return(et)
|
216
|
+
end
|
217
|
+
subject.multi_receive(events)
|
218
|
+
end
|
219
|
+
|
220
|
+
end
|
221
|
+
|
222
|
+
context "429 errors" do
|
223
|
+
let(:event) { ::LogStash::Event.new("foo" => "bar") }
|
224
|
+
let(:error) do
|
225
|
+
::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
|
226
|
+
429, double("url").as_null_object, double("request body"), double("response body")
|
227
|
+
)
|
228
|
+
end
|
229
|
+
let(:logger) { double("logger").as_null_object }
|
230
|
+
let(:response) { { :errors => [], :items => [] } }
|
231
|
+
|
232
|
+
before(:each) do
|
233
|
+
|
234
|
+
i = 0
|
235
|
+
bulk_param = [["index", anything, event.to_hash]]
|
236
|
+
|
237
|
+
allow(subject).to receive(:logger).and_return(logger)
|
238
|
+
|
239
|
+
# Fail the first time bulk is called, succeed the next time
|
240
|
+
allow(subject.client).to receive(:bulk).with(bulk_param) do
|
241
|
+
i += 1
|
242
|
+
if i == 1
|
243
|
+
raise error
|
244
|
+
end
|
245
|
+
end.and_return(response)
|
246
|
+
subject.multi_receive([event])
|
247
|
+
end
|
248
|
+
|
249
|
+
it "should retry the 429 till it goes away" do
|
250
|
+
expect(subject.client).to have_received(:bulk).twice
|
251
|
+
end
|
252
|
+
|
253
|
+
it "should log a debug message" do
|
254
|
+
expect(subject.logger).to have_received(:debug).with(/Encountered a retryable error/i, anything)
|
255
|
+
end
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
context "with timeout set" do
|
260
|
+
let(:listener) { Flores::Random.tcp_listener }
|
261
|
+
let(:port) { listener[2] }
|
262
|
+
let(:options) do
|
263
|
+
{
|
264
|
+
"manage_template" => false,
|
265
|
+
"hosts" => "localhost:#{port}",
|
266
|
+
"timeout" => 0.1, # fast timeout,
|
267
|
+
"port" => listener[2],
|
268
|
+
"protocol" => "http",
|
269
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
270
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
271
|
+
}
|
272
|
+
end
|
273
|
+
|
274
|
+
before do
|
275
|
+
# Expect a timeout to be logged.
|
276
|
+
expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request to Elasticsearch/i, anything).at_least(:once)
|
277
|
+
expect(subject.client).to receive(:bulk).at_least(:twice).and_call_original
|
278
|
+
end
|
279
|
+
|
280
|
+
it "should fail after the timeout" do
|
281
|
+
#pending("This is tricky now that we do healthchecks on instantiation")
|
282
|
+
Thread.new { subject.multi_receive([LogStash::Event.new]) }
|
283
|
+
|
284
|
+
# Allow the timeout to occur
|
285
|
+
sleep 6
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|
289
|
+
describe "the action option" do
|
290
|
+
context "with a sprintf action" do
|
291
|
+
let(:options) { {"action" => "%{myactionfield}" , "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
292
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
293
|
+
|
294
|
+
let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
|
295
|
+
|
296
|
+
it "should interpolate the requested action value when creating an event_action_tuple" do
|
297
|
+
expect(subject.event_action_tuple(event).first).to eql("update")
|
298
|
+
end
|
299
|
+
end
|
300
|
+
|
301
|
+
context "with a sprintf action equals to update" do
|
302
|
+
let(:options) { {"action" => "%{myactionfield}", "upsert" => '{"message": "some text"}' ,
|
303
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
304
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
305
|
+
|
306
|
+
let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
|
307
|
+
|
308
|
+
it "should obtain specific action's params from event_action_tuple" do
|
309
|
+
expect(subject.event_action_tuple(event)[1]).to include(:_upsert)
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
313
|
+
context "with an invalid action" do
|
314
|
+
let(:options) { {"action" => "SOME Garbaaage",
|
315
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
316
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
317
|
+
let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
|
318
|
+
|
319
|
+
it "should raise a configuration error" do
|
320
|
+
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
321
|
+
end
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
describe "SSL end to end" do
|
326
|
+
let(:do_register) { false } # skip the register in the global before block, as is called here.
|
327
|
+
let(:manticore_double) do
|
328
|
+
double("manticoreX#{self.inspect}")
|
329
|
+
end
|
330
|
+
|
331
|
+
before(:each) do
|
332
|
+
response_double = double("manticore response").as_null_object
|
333
|
+
# Allow healtchecks
|
334
|
+
allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
|
335
|
+
allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
|
336
|
+
allow(manticore_double).to receive(:close)
|
337
|
+
|
338
|
+
allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
|
339
|
+
subject.register
|
340
|
+
end
|
341
|
+
|
342
|
+
shared_examples("an encrypted client connection") do
|
343
|
+
it "should enable SSL in manticore" do
|
344
|
+
expect(subject.client.pool.urls.map(&:scheme).uniq).to eql(['https'])
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
|
349
|
+
context "With the 'ssl' option" do
|
350
|
+
let(:options) { {"ssl" => true,
|
351
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
352
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}}
|
353
|
+
|
354
|
+
include_examples("an encrypted client connection")
|
355
|
+
end
|
356
|
+
|
357
|
+
context "With an https host" do
|
358
|
+
let(:options) { {"hosts" => "https://localhost",
|
359
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
360
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
361
|
+
include_examples("an encrypted client connection")
|
362
|
+
end
|
363
|
+
end
|
364
|
+
|
365
|
+
describe "retry_on_conflict" do
|
366
|
+
let(:num_retries) { 123 }
|
367
|
+
let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
|
368
|
+
let(:options) { { 'retry_on_conflict' => num_retries ,
|
369
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
370
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
371
|
+
|
372
|
+
context "with a regular index" do
|
373
|
+
let(:options) { super.merge("action" => "index") }
|
374
|
+
|
375
|
+
it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do
|
376
|
+
allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
|
377
|
+
action, params, event_data = subject.event_action_tuple(event)
|
378
|
+
expect(params).not_to include({:_retry_on_conflict => num_retries})
|
379
|
+
end
|
380
|
+
end
|
381
|
+
|
382
|
+
context "using a plain update" do
|
383
|
+
let(:options) { super.merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
|
384
|
+
|
385
|
+
it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
|
386
|
+
action, params, event_data = subject.event_action_tuple(event)
|
387
|
+
expect(params).to include({:_retry_on_conflict => num_retries})
|
388
|
+
end
|
389
|
+
end
|
390
|
+
|
391
|
+
context "with a sprintf action that resolves to update" do
|
392
|
+
let(:options) { super.merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
|
393
|
+
|
394
|
+
it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
|
395
|
+
action, params, event_data = subject.event_action_tuple(event)
|
396
|
+
expect(params).to include({:_retry_on_conflict => num_retries})
|
397
|
+
expect(action).to eq("update")
|
398
|
+
end
|
399
|
+
end
|
400
|
+
end
|
401
|
+
|
402
|
+
describe "sleep interval calculation" do
|
403
|
+
let(:retry_max_interval) { 64 }
|
404
|
+
let(:options) { { "retry_max_interval" => retry_max_interval ,
|
405
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
406
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
407
|
+
|
408
|
+
it "should double the given value" do
|
409
|
+
expect(subject.next_sleep_interval(2)).to eql(4)
|
410
|
+
expect(subject.next_sleep_interval(32)).to eql(64)
|
411
|
+
end
|
412
|
+
|
413
|
+
it "should not increase the value past the max retry interval" do
|
414
|
+
sleep_interval = 2
|
415
|
+
100.times do
|
416
|
+
sleep_interval = subject.next_sleep_interval(sleep_interval)
|
417
|
+
expect(sleep_interval).to be <= retry_max_interval
|
418
|
+
end
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
422
|
+
describe "stale connection check" do
|
423
|
+
let(:validate_after_inactivity) { 123 }
|
424
|
+
let(:options) { { "validate_after_inactivity" => validate_after_inactivity ,
|
425
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
426
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
427
|
+
let(:do_register) { false }
|
428
|
+
|
429
|
+
before :each do
|
430
|
+
allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
|
431
|
+
end
|
432
|
+
|
433
|
+
after :each do
|
434
|
+
subject.close
|
435
|
+
end
|
436
|
+
|
437
|
+
it "should set the correct http client option for 'validate_after_inactivity'" do
|
438
|
+
subject.register
|
439
|
+
expect(::Manticore::Client).to have_received(:new) do |options|
|
440
|
+
expect(options[:check_connection_timeout]).to eq(validate_after_inactivity)
|
441
|
+
end
|
442
|
+
end
|
443
|
+
end
|
444
|
+
|
445
|
+
describe "custom parameters" do
|
446
|
+
|
447
|
+
let(:manticore_urls) { subject.client.pool.urls }
|
448
|
+
let(:manticore_url) { manticore_urls.first }
|
449
|
+
|
450
|
+
let(:custom_parameters_hash) { { "id" => 1, "name" => "logstash" } }
|
451
|
+
let(:custom_parameters_query) { custom_parameters_hash.map {|k,v| "#{k}=#{v}" }.join("&") }
|
452
|
+
|
453
|
+
context "using non-url hosts" do
|
454
|
+
|
455
|
+
let(:options) {
|
456
|
+
{
|
457
|
+
"index" => "my-index",
|
458
|
+
"hosts" => ["localhost:9200"],
|
459
|
+
"path" => "some-path",
|
460
|
+
"parameters" => custom_parameters_hash,
|
461
|
+
"port" => 9200,
|
462
|
+
"protocol" => "http",
|
463
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
464
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
465
|
+
}
|
466
|
+
}
|
467
|
+
|
468
|
+
it "creates a URI with the added parameters" do
|
469
|
+
expect(subject.parameters).to eql(custom_parameters_hash)
|
470
|
+
end
|
471
|
+
|
472
|
+
it "sets the query string on the HTTP client" do
|
473
|
+
expect(manticore_url.query).to eql(custom_parameters_query)
|
474
|
+
end
|
475
|
+
end
|
476
|
+
|
477
|
+
context "using url hosts" do
|
478
|
+
|
479
|
+
context "with embedded query parameters" do
|
480
|
+
let(:options) {
|
481
|
+
{ "hosts" => ["http://localhost:9200/path?#{custom_parameters_query}"] ,
|
482
|
+
"port" => 9200,
|
483
|
+
"protocol" => "http",
|
484
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
485
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
|
486
|
+
}
|
487
|
+
|
488
|
+
it "sets the query string on the HTTP client" do
|
489
|
+
expect(manticore_url.query).to eql(custom_parameters_query)
|
490
|
+
end
|
491
|
+
end
|
492
|
+
|
493
|
+
context "with explicit query parameters" do
|
494
|
+
let(:options) {
|
495
|
+
{
|
496
|
+
"hosts" => ["http://localhost:9200/path"],
|
497
|
+
"parameters" => custom_parameters_hash,
|
498
|
+
"port" => 9200,
|
499
|
+
"protocol" => "http",
|
500
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
501
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
502
|
+
}
|
503
|
+
}
|
504
|
+
|
505
|
+
it "sets the query string on the HTTP client" do
|
506
|
+
expect(manticore_url.query).to eql(custom_parameters_query)
|
507
|
+
end
|
508
|
+
end
|
509
|
+
|
510
|
+
context "with explicit query parameters and existing url parameters" do
|
511
|
+
let(:existing_query_string) { "existing=param" }
|
512
|
+
let(:options) {
|
513
|
+
{
|
514
|
+
"hosts" => ["http://localhost:9200/path?#{existing_query_string}"],
|
515
|
+
"parameters" => custom_parameters_hash,
|
516
|
+
"port" => 9200,
|
517
|
+
"protocol" => "http",
|
518
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
519
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
520
|
+
}
|
521
|
+
}
|
522
|
+
|
523
|
+
it "keeps the existing query string" do
|
524
|
+
expect(manticore_url.query).to include(existing_query_string)
|
525
|
+
end
|
526
|
+
|
527
|
+
it "includes the new query string" do
|
528
|
+
expect(manticore_url.query).to include(custom_parameters_query)
|
529
|
+
end
|
530
|
+
|
531
|
+
it "appends the new query string to the existing one" do
|
532
|
+
expect(manticore_url.query).to eql("#{existing_query_string}&#{custom_parameters_query}")
|
533
|
+
end
|
534
|
+
end
|
535
|
+
end
|
536
|
+
end
|
537
|
+
|
538
|
+
context 'handling amazon_es document-level status meant for the DLQ' do
|
539
|
+
let(:options) { { "manage_template" => false ,
|
540
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
541
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
542
|
+
let(:logger) { subject.instance_variable_get(:@logger) }
|
543
|
+
|
544
|
+
context 'when @dlq_writer is nil' do
|
545
|
+
before { subject.instance_variable_set '@dlq_writer', nil }
|
546
|
+
|
547
|
+
context 'resorting to previous behaviour of logging the error' do
|
548
|
+
context 'getting an invalid_index_name_exception' do
|
549
|
+
it 'should log at ERROR level' do
|
550
|
+
expect(logger).to receive(:error).with(/Could not index/, hash_including(:status, :action, :response))
|
551
|
+
mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } }
|
552
|
+
subject.handle_dlq_status("Could not index event to Elasticsearch.",
|
553
|
+
[:action, :params, :event], :some_status, mock_response)
|
554
|
+
end
|
555
|
+
end
|
556
|
+
|
557
|
+
context 'when getting any other exception' do
|
558
|
+
it 'should log at WARN level' do
|
559
|
+
expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
|
560
|
+
mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
|
561
|
+
subject.handle_dlq_status("Could not index event to Elasticsearch.",
|
562
|
+
[:action, :params, :event], :some_status, mock_response)
|
563
|
+
end
|
564
|
+
end
|
565
|
+
|
566
|
+
context 'when the response does not include [error]' do
|
567
|
+
it 'should not fail, but just log a warning' do
|
568
|
+
expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
|
569
|
+
mock_response = { 'index' => {} }
|
570
|
+
expect do
|
571
|
+
subject.handle_dlq_status("Could not index event to Elasticsearch.",
|
572
|
+
[:action, :params, :event], :some_status, mock_response)
|
573
|
+
end.to_not raise_error
|
574
|
+
end
|
575
|
+
end
|
576
|
+
end
|
577
|
+
end
|
578
|
+
|
579
|
+
# DLQ writer always nil, no matter what I try here. So mocking it all the way
|
580
|
+
context 'when DLQ is enabled' do
|
581
|
+
let(:dlq_writer) { double('DLQ writer') }
|
582
|
+
before { subject.instance_variable_set('@dlq_writer', dlq_writer) }
|
583
|
+
|
584
|
+
# Note: This is not quite the desired behaviour.
|
585
|
+
# We should still log when sending to the DLQ.
|
586
|
+
# This shall be solved by another issue, however: logstash-output-amazon_es#772
|
587
|
+
it 'should send the event to the DLQ instead, and not log' do
|
588
|
+
expect(dlq_writer).to receive(:write).with(:event, /Could not index/)
|
589
|
+
mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
|
590
|
+
subject.handle_dlq_status("Could not index event to Elasticsearch.",
|
591
|
+
[:action, :params, :event], :some_status, mock_response)
|
592
|
+
end
|
593
|
+
end
|
594
|
+
end
|
595
|
+
|
596
|
+
describe "custom headers" do
|
597
|
+
let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) }
|
598
|
+
|
599
|
+
context "when set" do
|
600
|
+
let(:headers) { { "X-Thing" => "Test" } }
|
601
|
+
let(:options) { { "custom_headers" => headers ,
|
602
|
+
"aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
|
603
|
+
"aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
|
604
|
+
it "should use the custom headers in the adapter options" do
|
605
|
+
expect(manticore_options[:headers]).to eq(headers)
|
606
|
+
end
|
607
|
+
end
|
608
|
+
|
609
|
+
context "when not set" do
|
610
|
+
it "should have no headers" do
|
611
|
+
expect(manticore_options[:headers]).to be_empty
|
612
|
+
end
|
613
|
+
end
|
614
|
+
end
|
615
|
+
end
|