logstash-output-elasticsearch 0.1.6 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -13
- data/CHANGELOG.md +117 -0
- data/CONTRIBUTORS +32 -0
- data/Gemfile +4 -4
- data/LICENSE +1 -1
- data/NOTICE.TXT +5 -0
- data/README.md +110 -0
- data/lib/logstash/outputs/elasticsearch.rb +97 -425
- data/lib/logstash/outputs/elasticsearch/buffer.rb +124 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +205 -0
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +164 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json +36 -24
- data/lib/logstash/outputs/elasticsearch/http_client.rb +236 -0
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +106 -0
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +35 -0
- data/logstash-output-elasticsearch.gemspec +17 -15
- data/spec/es_spec_helper.rb +77 -0
- data/spec/fixtures/scripts/scripted_update.groovy +2 -0
- data/spec/fixtures/scripts/scripted_update_nested.groovy +2 -0
- data/spec/fixtures/scripts/scripted_upsert.groovy +2 -0
- data/spec/integration/outputs/create_spec.rb +55 -0
- data/spec/integration/outputs/index_spec.rb +68 -0
- data/spec/integration/outputs/parent_spec.rb +73 -0
- data/spec/integration/outputs/pipeline_spec.rb +75 -0
- data/spec/integration/outputs/retry_spec.rb +163 -0
- data/spec/integration/outputs/routing_spec.rb +65 -0
- data/spec/integration/outputs/secure_spec.rb +108 -0
- data/spec/integration/outputs/templates_spec.rb +90 -0
- data/spec/integration/outputs/update_spec.rb +188 -0
- data/spec/unit/buffer_spec.rb +118 -0
- data/spec/unit/http_client_builder_spec.rb +27 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +133 -0
- data/spec/unit/outputs/elasticsearch_proxy_spec.rb +58 -0
- data/spec/unit/outputs/elasticsearch_spec.rb +227 -0
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +55 -0
- metadata +137 -51
- data/.gitignore +0 -4
- data/Rakefile +0 -6
- data/lib/logstash/outputs/elasticsearch/protocol.rb +0 -253
- data/rakelib/publish.rake +0 -9
- data/rakelib/vendor.rake +0 -169
- data/spec/outputs/elasticsearch.rb +0 -518
@@ -0,0 +1,118 @@
|
|
1
|
+
require "logstash/outputs/elasticsearch"
|
2
|
+
require "logstash/outputs/elasticsearch/buffer"
|
3
|
+
|
4
|
+
describe LogStash::Outputs::ElasticSearch::Buffer do
|
5
|
+
class OperationTarget # Used to track buffer flushesn
|
6
|
+
attr_reader :buffer, :buffer_history, :receive_count
|
7
|
+
def initialize
|
8
|
+
@buffer = nil
|
9
|
+
@buffer_history = []
|
10
|
+
@receive_count = 0
|
11
|
+
end
|
12
|
+
|
13
|
+
def receive(buffer)
|
14
|
+
@receive_count += 1
|
15
|
+
@buffer_history << buffer.clone
|
16
|
+
@buffer = buffer
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
let(:logger) { Cabin::Channel.get }
|
21
|
+
let(:max_size) { 10 }
|
22
|
+
let(:flush_interval) { 2 }
|
23
|
+
# Used to track flush count
|
24
|
+
let(:operation_target) { OperationTarget.new() }
|
25
|
+
let(:operation) { proc {|buffer| operation_target.receive(buffer) } }
|
26
|
+
subject(:buffer){ LogStash::Outputs::ElasticSearch::Buffer.new(logger, max_size, flush_interval, &operation) }
|
27
|
+
|
28
|
+
after(:each) do
|
29
|
+
buffer.stop(do_flush=false)
|
30
|
+
end
|
31
|
+
|
32
|
+
it "should initialize cleanly" do
|
33
|
+
expect(buffer).to be_a(LogStash::Outputs::ElasticSearch::Buffer)
|
34
|
+
end
|
35
|
+
|
36
|
+
shared_examples("a buffer with two items inside") do
|
37
|
+
it "should add a pushed item to the buffer" do
|
38
|
+
buffer.synchronize do |data|
|
39
|
+
expect(data).to include(item1)
|
40
|
+
expect(data).to include(item2)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe "interval flushing" do
|
45
|
+
before do
|
46
|
+
sleep flush_interval + 1
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should flush the buffer after the interval has passed" do
|
50
|
+
expect(operation_target.receive_count).to eql(1)
|
51
|
+
end
|
52
|
+
|
53
|
+
it "should clear the buffer after a successful flush" do
|
54
|
+
expect(operation_target.buffer).to eql([])
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
describe "interval flushing a stopped buffer" do
|
59
|
+
before do
|
60
|
+
buffer.stop(do_flush=false)
|
61
|
+
sleep flush_interval + 1
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should not flush if the buffer is stopped" do
|
65
|
+
expect(operation_target.receive_count).to eql(0)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
describe "with a buffer push" do
|
71
|
+
let(:item1) { "foo" }
|
72
|
+
let(:item2) { "bar" }
|
73
|
+
|
74
|
+
describe "a buffer with two items pushed to it separately" do
|
75
|
+
before do
|
76
|
+
buffer << item1
|
77
|
+
buffer << item2
|
78
|
+
end
|
79
|
+
|
80
|
+
include_examples("a buffer with two items inside")
|
81
|
+
end
|
82
|
+
|
83
|
+
describe "a buffer with two items pushed to it in one operation" do
|
84
|
+
before do
|
85
|
+
buffer.push_multi([item1, item2])
|
86
|
+
end
|
87
|
+
|
88
|
+
include_examples("a buffer with two items inside")
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe "with an empty buffer" do
|
93
|
+
it "should not perform an operation if the buffer is empty" do
|
94
|
+
buffer.flush
|
95
|
+
expect(operation_target.receive_count).to eql(0)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
describe "flushing with an operation that raises an error" do
|
100
|
+
class TestError < StandardError; end
|
101
|
+
let(:operation) { proc {|buffer| raise TestError, "A test" } }
|
102
|
+
let(:item) { double("item") }
|
103
|
+
|
104
|
+
before do
|
105
|
+
buffer << item
|
106
|
+
end
|
107
|
+
|
108
|
+
it "should raise an exception" do
|
109
|
+
expect { buffer.flush }.to raise_error(TestError)
|
110
|
+
end
|
111
|
+
|
112
|
+
it "should not clear the buffer" do
|
113
|
+
expect do
|
114
|
+
buffer.flush rescue TestError
|
115
|
+
end.not_to change(buffer, :contents)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require "logstash/outputs/elasticsearch"
|
2
|
+
require "logstash/outputs/elasticsearch/http_client"
|
3
|
+
require "logstash/outputs/elasticsearch/http_client_builder"
|
4
|
+
|
5
|
+
describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
|
6
|
+
describe "auth setup with url encodable passwords" do
|
7
|
+
let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
|
8
|
+
let(:user) { "foo@bar"}
|
9
|
+
let(:password) {"baz@blah" }
|
10
|
+
let(:password_secured) do
|
11
|
+
secured = double("password")
|
12
|
+
allow(secured).to receive(:value).and_return(password)
|
13
|
+
secured
|
14
|
+
end
|
15
|
+
let(:options) { {"user" => user, "password" => password} }
|
16
|
+
let(:logger) { mock("logger") }
|
17
|
+
let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
|
18
|
+
|
19
|
+
it "should return the user verbatim" do
|
20
|
+
expect(auth_setup[:user]).to eql(user)
|
21
|
+
end
|
22
|
+
|
23
|
+
it "should return the password verbatim" do
|
24
|
+
expect(auth_setup[:password]).to eql(password)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/elasticsearch/http_client"
|
3
|
+
require "java"
|
4
|
+
|
5
|
+
describe LogStash::Outputs::ElasticSearch::HttpClient do
|
6
|
+
let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get }}
|
7
|
+
|
8
|
+
describe "Host/URL Parsing" do
|
9
|
+
subject { described_class.new(base_options) }
|
10
|
+
|
11
|
+
let(:true_hostname) { "my-dash.hostname" }
|
12
|
+
let(:ipv6_hostname) { "[::1]" }
|
13
|
+
let(:ipv4_hostname) { "127.0.0.1" }
|
14
|
+
let(:port) { 9202 }
|
15
|
+
let(:hostname_port) { "#{hostname}:#{port}"}
|
16
|
+
let(:http_hostname_port) { "http://#{hostname_port}"}
|
17
|
+
let(:https_hostname_port) { "https://#{hostname_port}"}
|
18
|
+
let(:http_hostname_port_path) { "http://#{hostname_port}/path"}
|
19
|
+
|
20
|
+
shared_examples("proper host handling") do
|
21
|
+
it "should properly transform a host:port string to a URL" do
|
22
|
+
expect(subject.send(:host_to_url, hostname_port)).to eql(http_hostname_port)
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should raise an error when a partial URL is an invalid format" do
|
26
|
+
expect {
|
27
|
+
subject.send(:host_to_url, "#{hostname_port}/")
|
28
|
+
}.to raise_error(LogStash::ConfigurationError)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should not raise an error with a / for a path" do
|
32
|
+
expect(subject.send(:host_to_url, "#{http_hostname_port}/")).to eql("#{http_hostname_port}/")
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should parse full URLs correctly" do
|
36
|
+
expect(subject.send(:host_to_url, http_hostname_port)).to eql(http_hostname_port)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should reject full URLs with usernames and passwords" do
|
40
|
+
expect {
|
41
|
+
subject.send(:host_to_url, "http://user:password@host.domain")
|
42
|
+
}.to raise_error(LogStash::ConfigurationError)
|
43
|
+
end
|
44
|
+
|
45
|
+
describe "ssl" do
|
46
|
+
it "should refuse to handle an http url when ssl is true" do
|
47
|
+
expect {
|
48
|
+
subject.send(:host_to_url, http_hostname_port, true)
|
49
|
+
}.to raise_error(LogStash::ConfigurationError)
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should refuse to handle an https url when ssl is false" do
|
53
|
+
expect {
|
54
|
+
subject.send(:host_to_url, https_hostname_port, false)
|
55
|
+
}.to raise_error(LogStash::ConfigurationError)
|
56
|
+
end
|
57
|
+
|
58
|
+
it "should handle an ssl url correctly when SSL is nil" do
|
59
|
+
expect(subject.send(:host_to_url, https_hostname_port, nil)).to eql(https_hostname_port)
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should raise an exception if an unexpected value is passed in" do
|
63
|
+
expect { subject.send(:host_to_url, https_hostname_port, {})}.to raise_error(ArgumentError)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe "path" do
|
68
|
+
it "should allow paths in a url" do
|
69
|
+
expect(subject.send(:host_to_url, http_hostname_port_path, nil)).to eql(http_hostname_port_path)
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should not allow paths in two places" do
|
73
|
+
expect {
|
74
|
+
subject.send(:host_to_url, http_hostname_port_path, false, "/otherpath")
|
75
|
+
}.to raise_error(LogStash::ConfigurationError)
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should automatically insert a / in front of path overlays if needed" do
|
79
|
+
expect(subject.send(:host_to_url, http_hostname_port, false, "otherpath")).to eql(http_hostname_port + "/otherpath")
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
describe "an regular hostname" do
|
85
|
+
let(:hostname) { true_hostname }
|
86
|
+
include_examples("proper host handling")
|
87
|
+
end
|
88
|
+
|
89
|
+
describe "an ipv4 host" do
|
90
|
+
let(:hostname) { ipv4_hostname }
|
91
|
+
include_examples("proper host handling")
|
92
|
+
end
|
93
|
+
|
94
|
+
describe "an ipv6 host" do
|
95
|
+
let(:hostname) { ipv6_hostname }
|
96
|
+
include_examples("proper host handling")
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
describe "sniffing" do
|
101
|
+
let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
|
102
|
+
let(:transport) { client.client.transport }
|
103
|
+
|
104
|
+
before do
|
105
|
+
allow(transport).to receive(:reload_connections!)
|
106
|
+
end
|
107
|
+
|
108
|
+
context "with sniffing enabled" do
|
109
|
+
let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
|
110
|
+
|
111
|
+
after do
|
112
|
+
client.stop_sniffing!
|
113
|
+
end
|
114
|
+
|
115
|
+
it "should start the sniffer" do
|
116
|
+
expect(client.sniffer_thread).to be_a(Thread)
|
117
|
+
end
|
118
|
+
|
119
|
+
it "should periodically sniff the client" do
|
120
|
+
sleep 2
|
121
|
+
expect(transport).to have_received(:reload_connections!).at_least(:once)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
context "with sniffing disabled" do
|
126
|
+
let(:client_opts) { {:sniffing => false} }
|
127
|
+
|
128
|
+
it "should not start the sniffer" do
|
129
|
+
expect(client.sniffer_thread).to be_nil
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require_relative "../../../spec/es_spec_helper"
|
2
|
+
require 'stud/temporary'
|
3
|
+
require 'elasticsearch'
|
4
|
+
require "logstash/outputs/elasticsearch"
|
5
|
+
|
6
|
+
describe "Proxy option" do
|
7
|
+
let(:settings) {
|
8
|
+
{
|
9
|
+
"hosts" => "node01",
|
10
|
+
"proxy" => proxy
|
11
|
+
}
|
12
|
+
}
|
13
|
+
subject {
|
14
|
+
LogStash::Outputs::ElasticSearch.new(settings)
|
15
|
+
}
|
16
|
+
|
17
|
+
before do
|
18
|
+
allow(::Elasticsearch::Client).to receive(:new).with(any_args)
|
19
|
+
end
|
20
|
+
|
21
|
+
describe "valid configs" do
|
22
|
+
before do
|
23
|
+
subject.register
|
24
|
+
end
|
25
|
+
|
26
|
+
context "when specified as a string" do
|
27
|
+
let(:proxy) { "http://127.0.0.1:1234" }
|
28
|
+
|
29
|
+
it "should set the proxy to the exact value" do
|
30
|
+
expect(::Elasticsearch::Client).to have_received(:new) do |options|
|
31
|
+
expect(options[:transport_options][:proxy]).to eql(proxy)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
context "when specified as a hash" do
|
37
|
+
let(:proxy) { {"hosts" => "127.0.0.1", "protocol" => "http"} }
|
38
|
+
|
39
|
+
it "should pass through the proxy values as symbols" do
|
40
|
+
expected = {:hosts => proxy["hosts"], :protocol => proxy["protocol"]}
|
41
|
+
expect(::Elasticsearch::Client).to have_received(:new) do |options|
|
42
|
+
expect(options[:transport_options][:proxy]).to eql(expected)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
describe "invalid configs" do
|
49
|
+
let(:proxy) { ["bad", "stuff"] }
|
50
|
+
|
51
|
+
it "should have raised an exception" do
|
52
|
+
expect {
|
53
|
+
subject.register
|
54
|
+
}.to raise_error(LogStash::ConfigurationError)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
@@ -0,0 +1,227 @@
|
|
1
|
+
require_relative "../../../spec/es_spec_helper"
|
2
|
+
require "flores/random"
|
3
|
+
require "logstash/outputs/elasticsearch"
|
4
|
+
|
5
|
+
describe "outputs/elasticsearch" do
|
6
|
+
context "with an active instance" do
|
7
|
+
let(:options) {
|
8
|
+
{
|
9
|
+
"index" => "my-index",
|
10
|
+
"hosts" => ["localhost","localhost:9202"],
|
11
|
+
"path" => "some-path"
|
12
|
+
}
|
13
|
+
}
|
14
|
+
|
15
|
+
let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
|
16
|
+
|
17
|
+
let(:manticore_host) {
|
18
|
+
eso.client.send(:client).transport.options[:hosts].first
|
19
|
+
}
|
20
|
+
|
21
|
+
around(:each) do |block|
|
22
|
+
eso.register
|
23
|
+
block.call()
|
24
|
+
eso.close
|
25
|
+
end
|
26
|
+
|
27
|
+
describe "getting a document type" do
|
28
|
+
it "should default to 'logs'" do
|
29
|
+
expect(eso.send(:get_event_type, LogStash::Event.new)).to eql("logs")
|
30
|
+
end
|
31
|
+
|
32
|
+
it "should get the type from the event if nothing else specified in the config" do
|
33
|
+
expect(eso.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
|
34
|
+
end
|
35
|
+
|
36
|
+
context "with 'document type set'" do
|
37
|
+
let(:options) { super.merge("document_type" => "bar")}
|
38
|
+
it "should get the event type from the 'document_type' setting" do
|
39
|
+
expect(eso.send(:get_event_type, LogStash::Event.new())).to eql("bar")
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
context "with a bad type" do
|
44
|
+
let(:type_arg) { ["foo"] }
|
45
|
+
let(:result) { eso.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
|
46
|
+
|
47
|
+
before do
|
48
|
+
allow(eso.instance_variable_get(:@logger)).to receive(:warn)
|
49
|
+
result
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should call @logger.warn and return nil" do
|
53
|
+
expect(eso.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should set the type to the stringified value" do
|
57
|
+
expect(result).to eql(type_arg.to_s)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
describe "with path" do
|
63
|
+
it "should properly create a URI with the path" do
|
64
|
+
expect(eso.path).to eql(options["path"])
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should properly set the path on the HTTP client adding slashes" do
|
68
|
+
expect(manticore_host).to include("/" + options["path"] + "/")
|
69
|
+
end
|
70
|
+
|
71
|
+
context "with extra slashes" do
|
72
|
+
let(:path) { "/slashed-path/ "}
|
73
|
+
let(:eso) {
|
74
|
+
LogStash::Outputs::ElasticSearch.new(options.merge("path" => "/some-path/"))
|
75
|
+
}
|
76
|
+
|
77
|
+
it "should properly set the path on the HTTP client without adding slashes" do
|
78
|
+
expect(manticore_host).to include(options["path"])
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
describe "without a port specified" do
|
83
|
+
it "should properly set the default port (9200) on the HTTP client" do
|
84
|
+
expect(manticore_host).to include("9200")
|
85
|
+
end
|
86
|
+
end
|
87
|
+
describe "with a port other than 9200 specified" do
|
88
|
+
let(:manticore_host) {
|
89
|
+
eso.client.send(:client).transport.options[:hosts].last
|
90
|
+
}
|
91
|
+
it "should properly set the specified port on the HTTP client" do
|
92
|
+
expect(manticore_host).to include("9202")
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
describe "#multi_receive" do
|
97
|
+
let(:events) { [double("one"), double("two"), double("three")] }
|
98
|
+
let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
|
99
|
+
let(:options) { super.merge("flush_size" => 2) }
|
100
|
+
|
101
|
+
before do
|
102
|
+
allow(eso).to receive(:retrying_submit).with(anything)
|
103
|
+
events.each_with_index do |e,i|
|
104
|
+
et = events_tuples[i]
|
105
|
+
allow(eso).to receive(:event_action_tuple).with(e).and_return(et)
|
106
|
+
end
|
107
|
+
eso.multi_receive(events)
|
108
|
+
end
|
109
|
+
|
110
|
+
it "should receive an array of events and invoke retrying_submit with them, split by flush_size" do
|
111
|
+
expect(eso).to have_received(:retrying_submit).with(events_tuples.slice(0,2))
|
112
|
+
expect(eso).to have_received(:retrying_submit).with(events_tuples.slice(2,3))
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
# TODO(sissel): Improve this. I'm not a fan of using message expectations (expect().to receive...)
|
120
|
+
# especially with respect to logging to verify a failure/retry has occurred. For now, this
|
121
|
+
# should suffice, though.
|
122
|
+
context "with timeout set" do
|
123
|
+
let(:listener) { Flores::Random.tcp_listener }
|
124
|
+
let(:port) { listener[2] }
|
125
|
+
let(:options) do
|
126
|
+
{
|
127
|
+
"manage_template" => false,
|
128
|
+
"hosts" => "localhost:#{port}",
|
129
|
+
"flush_size" => 1,
|
130
|
+
"timeout" => 0.1, # fast timeout
|
131
|
+
}
|
132
|
+
end
|
133
|
+
let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
|
134
|
+
|
135
|
+
before do
|
136
|
+
eso.register
|
137
|
+
|
138
|
+
# Expect a timeout to be logged.
|
139
|
+
expect(eso.logger).to receive(:error).with(/Attempted to send a bulk request/, anything)
|
140
|
+
end
|
141
|
+
|
142
|
+
after do
|
143
|
+
listener[0].close
|
144
|
+
# Stop the receive buffer, but don't flush because that would hang forever in this case since ES never returns a result
|
145
|
+
eso.instance_variable_get(:@buffer).stop(false,false)
|
146
|
+
eso.close
|
147
|
+
end
|
148
|
+
|
149
|
+
it "should fail after the timeout" do
|
150
|
+
Thread.new { eso.receive(LogStash::Event.new) }
|
151
|
+
|
152
|
+
# Allow the timeout to occur.
|
153
|
+
sleep(options["timeout"] + 0.5)
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
describe "the action option" do
|
158
|
+
subject(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
|
159
|
+
context "with a sprintf action" do
|
160
|
+
let(:options) { {"action" => "%{myactionfield}"} }
|
161
|
+
|
162
|
+
let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
|
163
|
+
|
164
|
+
it "should interpolate the requested action value when creating an event_action_tuple" do
|
165
|
+
expect(eso.event_action_tuple(event).first).to eql("update")
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
context "with an invalid action" do
|
170
|
+
let(:options) { {"action" => "SOME Garbaaage"} }
|
171
|
+
|
172
|
+
it "should raise a configuration error" do
|
173
|
+
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
describe "SSL end to end" do
|
179
|
+
shared_examples("an encrypted client connection") do
|
180
|
+
it "should enable SSL in manticore" do
|
181
|
+
expect(eso.client.client_options[:hosts].map {|h| URI.parse(h).scheme}.uniq).to eql(['https'])
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
|
186
|
+
subject(:manticore) { eso.client.client}
|
187
|
+
|
188
|
+
before do
|
189
|
+
eso.register
|
190
|
+
end
|
191
|
+
|
192
|
+
context "With the 'ssl' option" do
|
193
|
+
let(:options) { {"ssl" => true}}
|
194
|
+
|
195
|
+
include_examples("an encrypted client connection")
|
196
|
+
end
|
197
|
+
|
198
|
+
context "With an https host" do
|
199
|
+
let(:options) { {"hosts" => "https://localhost"} }
|
200
|
+
include_examples("an encrypted client connection")
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
describe "retry_on_conflict" do
|
205
|
+
let(:num_retries) { 123 }
|
206
|
+
let(:event) { LogStash::Event.new("message" => "blah") }
|
207
|
+
subject(:eso) {LogStash::Outputs::ElasticSearch.new(options.merge('retry_on_conflict' => num_retries))}
|
208
|
+
|
209
|
+
context "with a regular index" do
|
210
|
+
let(:options) { {"action" => "index"} }
|
211
|
+
|
212
|
+
it "should interpolate the requested action value when creating an event_action_tuple" do
|
213
|
+
action, params, event_data = eso.event_action_tuple(event)
|
214
|
+
expect(params).not_to include({:_retry_on_conflict => num_retries})
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
context "using a plain update" do
|
219
|
+
let(:options) { {"action" => "update", "retry_on_conflict" => num_retries} }
|
220
|
+
|
221
|
+
it "should interpolate the requested action value when creating an event_action_tuple" do
|
222
|
+
action, params, event_data = eso.event_action_tuple(event)
|
223
|
+
expect(params).to include({:_retry_on_conflict => num_retries})
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
end
|