logstash-output-elasticsearch 0.1.6 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +5 -13
  2. data/CHANGELOG.md +117 -0
  3. data/CONTRIBUTORS +32 -0
  4. data/Gemfile +4 -4
  5. data/LICENSE +1 -1
  6. data/NOTICE.TXT +5 -0
  7. data/README.md +110 -0
  8. data/lib/logstash/outputs/elasticsearch.rb +97 -425
  9. data/lib/logstash/outputs/elasticsearch/buffer.rb +124 -0
  10. data/lib/logstash/outputs/elasticsearch/common.rb +205 -0
  11. data/lib/logstash/outputs/elasticsearch/common_configs.rb +164 -0
  12. data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json +36 -24
  13. data/lib/logstash/outputs/elasticsearch/http_client.rb +236 -0
  14. data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +106 -0
  15. data/lib/logstash/outputs/elasticsearch/template_manager.rb +35 -0
  16. data/logstash-output-elasticsearch.gemspec +17 -15
  17. data/spec/es_spec_helper.rb +77 -0
  18. data/spec/fixtures/scripts/scripted_update.groovy +2 -0
  19. data/spec/fixtures/scripts/scripted_update_nested.groovy +2 -0
  20. data/spec/fixtures/scripts/scripted_upsert.groovy +2 -0
  21. data/spec/integration/outputs/create_spec.rb +55 -0
  22. data/spec/integration/outputs/index_spec.rb +68 -0
  23. data/spec/integration/outputs/parent_spec.rb +73 -0
  24. data/spec/integration/outputs/pipeline_spec.rb +75 -0
  25. data/spec/integration/outputs/retry_spec.rb +163 -0
  26. data/spec/integration/outputs/routing_spec.rb +65 -0
  27. data/spec/integration/outputs/secure_spec.rb +108 -0
  28. data/spec/integration/outputs/templates_spec.rb +90 -0
  29. data/spec/integration/outputs/update_spec.rb +188 -0
  30. data/spec/unit/buffer_spec.rb +118 -0
  31. data/spec/unit/http_client_builder_spec.rb +27 -0
  32. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +133 -0
  33. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +58 -0
  34. data/spec/unit/outputs/elasticsearch_spec.rb +227 -0
  35. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +55 -0
  36. metadata +137 -51
  37. data/.gitignore +0 -4
  38. data/Rakefile +0 -6
  39. data/lib/logstash/outputs/elasticsearch/protocol.rb +0 -253
  40. data/rakelib/publish.rake +0 -9
  41. data/rakelib/vendor.rake +0 -169
  42. data/spec/outputs/elasticsearch.rb +0 -518
@@ -0,0 +1,118 @@
1
+ require "logstash/outputs/elasticsearch"
2
+ require "logstash/outputs/elasticsearch/buffer"
3
+
4
+ describe LogStash::Outputs::ElasticSearch::Buffer do
5
+ class OperationTarget # Used to track buffer flushesn
6
+ attr_reader :buffer, :buffer_history, :receive_count
7
+ def initialize
8
+ @buffer = nil
9
+ @buffer_history = []
10
+ @receive_count = 0
11
+ end
12
+
13
+ def receive(buffer)
14
+ @receive_count += 1
15
+ @buffer_history << buffer.clone
16
+ @buffer = buffer
17
+ end
18
+ end
19
+
20
+ let(:logger) { Cabin::Channel.get }
21
+ let(:max_size) { 10 }
22
+ let(:flush_interval) { 2 }
23
+ # Used to track flush count
24
+ let(:operation_target) { OperationTarget.new() }
25
+ let(:operation) { proc {|buffer| operation_target.receive(buffer) } }
26
+ subject(:buffer){ LogStash::Outputs::ElasticSearch::Buffer.new(logger, max_size, flush_interval, &operation) }
27
+
28
+ after(:each) do
29
+ buffer.stop(do_flush=false)
30
+ end
31
+
32
+ it "should initialize cleanly" do
33
+ expect(buffer).to be_a(LogStash::Outputs::ElasticSearch::Buffer)
34
+ end
35
+
36
+ shared_examples("a buffer with two items inside") do
37
+ it "should add a pushed item to the buffer" do
38
+ buffer.synchronize do |data|
39
+ expect(data).to include(item1)
40
+ expect(data).to include(item2)
41
+ end
42
+ end
43
+
44
+ describe "interval flushing" do
45
+ before do
46
+ sleep flush_interval + 1
47
+ end
48
+
49
+ it "should flush the buffer after the interval has passed" do
50
+ expect(operation_target.receive_count).to eql(1)
51
+ end
52
+
53
+ it "should clear the buffer after a successful flush" do
54
+ expect(operation_target.buffer).to eql([])
55
+ end
56
+ end
57
+
58
+ describe "interval flushing a stopped buffer" do
59
+ before do
60
+ buffer.stop(do_flush=false)
61
+ sleep flush_interval + 1
62
+ end
63
+
64
+ it "should not flush if the buffer is stopped" do
65
+ expect(operation_target.receive_count).to eql(0)
66
+ end
67
+ end
68
+ end
69
+
70
+ describe "with a buffer push" do
71
+ let(:item1) { "foo" }
72
+ let(:item2) { "bar" }
73
+
74
+ describe "a buffer with two items pushed to it separately" do
75
+ before do
76
+ buffer << item1
77
+ buffer << item2
78
+ end
79
+
80
+ include_examples("a buffer with two items inside")
81
+ end
82
+
83
+ describe "a buffer with two items pushed to it in one operation" do
84
+ before do
85
+ buffer.push_multi([item1, item2])
86
+ end
87
+
88
+ include_examples("a buffer with two items inside")
89
+ end
90
+ end
91
+
92
+ describe "with an empty buffer" do
93
+ it "should not perform an operation if the buffer is empty" do
94
+ buffer.flush
95
+ expect(operation_target.receive_count).to eql(0)
96
+ end
97
+ end
98
+
99
+ describe "flushing with an operation that raises an error" do
100
+ class TestError < StandardError; end
101
+ let(:operation) { proc {|buffer| raise TestError, "A test" } }
102
+ let(:item) { double("item") }
103
+
104
+ before do
105
+ buffer << item
106
+ end
107
+
108
+ it "should raise an exception" do
109
+ expect { buffer.flush }.to raise_error(TestError)
110
+ end
111
+
112
+ it "should not clear the buffer" do
113
+ expect do
114
+ buffer.flush rescue TestError
115
+ end.not_to change(buffer, :contents)
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,27 @@
1
+ require "logstash/outputs/elasticsearch"
2
+ require "logstash/outputs/elasticsearch/http_client"
3
+ require "logstash/outputs/elasticsearch/http_client_builder"
4
+
5
+ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
6
+ describe "auth setup with url encodable passwords" do
7
+ let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
8
+ let(:user) { "foo@bar"}
9
+ let(:password) {"baz@blah" }
10
+ let(:password_secured) do
11
+ secured = double("password")
12
+ allow(secured).to receive(:value).and_return(password)
13
+ secured
14
+ end
15
+ let(:options) { {"user" => user, "password" => password} }
16
+ let(:logger) { mock("logger") }
17
+ let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
18
+
19
+ it "should return the user verbatim" do
20
+ expect(auth_setup[:user]).to eql(user)
21
+ end
22
+
23
+ it "should return the password verbatim" do
24
+ expect(auth_setup[:password]).to eql(password)
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,133 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/elasticsearch/http_client"
3
+ require "java"
4
+
5
+ describe LogStash::Outputs::ElasticSearch::HttpClient do
6
+ let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get }}
7
+
8
+ describe "Host/URL Parsing" do
9
+ subject { described_class.new(base_options) }
10
+
11
+ let(:true_hostname) { "my-dash.hostname" }
12
+ let(:ipv6_hostname) { "[::1]" }
13
+ let(:ipv4_hostname) { "127.0.0.1" }
14
+ let(:port) { 9202 }
15
+ let(:hostname_port) { "#{hostname}:#{port}"}
16
+ let(:http_hostname_port) { "http://#{hostname_port}"}
17
+ let(:https_hostname_port) { "https://#{hostname_port}"}
18
+ let(:http_hostname_port_path) { "http://#{hostname_port}/path"}
19
+
20
+ shared_examples("proper host handling") do
21
+ it "should properly transform a host:port string to a URL" do
22
+ expect(subject.send(:host_to_url, hostname_port)).to eql(http_hostname_port)
23
+ end
24
+
25
+ it "should raise an error when a partial URL is an invalid format" do
26
+ expect {
27
+ subject.send(:host_to_url, "#{hostname_port}/")
28
+ }.to raise_error(LogStash::ConfigurationError)
29
+ end
30
+
31
+ it "should not raise an error with a / for a path" do
32
+ expect(subject.send(:host_to_url, "#{http_hostname_port}/")).to eql("#{http_hostname_port}/")
33
+ end
34
+
35
+ it "should parse full URLs correctly" do
36
+ expect(subject.send(:host_to_url, http_hostname_port)).to eql(http_hostname_port)
37
+ end
38
+
39
+ it "should reject full URLs with usernames and passwords" do
40
+ expect {
41
+ subject.send(:host_to_url, "http://user:password@host.domain")
42
+ }.to raise_error(LogStash::ConfigurationError)
43
+ end
44
+
45
+ describe "ssl" do
46
+ it "should refuse to handle an http url when ssl is true" do
47
+ expect {
48
+ subject.send(:host_to_url, http_hostname_port, true)
49
+ }.to raise_error(LogStash::ConfigurationError)
50
+ end
51
+
52
+ it "should refuse to handle an https url when ssl is false" do
53
+ expect {
54
+ subject.send(:host_to_url, https_hostname_port, false)
55
+ }.to raise_error(LogStash::ConfigurationError)
56
+ end
57
+
58
+ it "should handle an ssl url correctly when SSL is nil" do
59
+ expect(subject.send(:host_to_url, https_hostname_port, nil)).to eql(https_hostname_port)
60
+ end
61
+
62
+ it "should raise an exception if an unexpected value is passed in" do
63
+ expect { subject.send(:host_to_url, https_hostname_port, {})}.to raise_error(ArgumentError)
64
+ end
65
+ end
66
+
67
+ describe "path" do
68
+ it "should allow paths in a url" do
69
+ expect(subject.send(:host_to_url, http_hostname_port_path, nil)).to eql(http_hostname_port_path)
70
+ end
71
+
72
+ it "should not allow paths in two places" do
73
+ expect {
74
+ subject.send(:host_to_url, http_hostname_port_path, false, "/otherpath")
75
+ }.to raise_error(LogStash::ConfigurationError)
76
+ end
77
+
78
+ it "should automatically insert a / in front of path overlays if needed" do
79
+ expect(subject.send(:host_to_url, http_hostname_port, false, "otherpath")).to eql(http_hostname_port + "/otherpath")
80
+ end
81
+ end
82
+ end
83
+
84
+ describe "an regular hostname" do
85
+ let(:hostname) { true_hostname }
86
+ include_examples("proper host handling")
87
+ end
88
+
89
+ describe "an ipv4 host" do
90
+ let(:hostname) { ipv4_hostname }
91
+ include_examples("proper host handling")
92
+ end
93
+
94
+ describe "an ipv6 host" do
95
+ let(:hostname) { ipv6_hostname }
96
+ include_examples("proper host handling")
97
+ end
98
+ end
99
+
100
+ describe "sniffing" do
101
+ let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
102
+ let(:transport) { client.client.transport }
103
+
104
+ before do
105
+ allow(transport).to receive(:reload_connections!)
106
+ end
107
+
108
+ context "with sniffing enabled" do
109
+ let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
110
+
111
+ after do
112
+ client.stop_sniffing!
113
+ end
114
+
115
+ it "should start the sniffer" do
116
+ expect(client.sniffer_thread).to be_a(Thread)
117
+ end
118
+
119
+ it "should periodically sniff the client" do
120
+ sleep 2
121
+ expect(transport).to have_received(:reload_connections!).at_least(:once)
122
+ end
123
+ end
124
+
125
+ context "with sniffing disabled" do
126
+ let(:client_opts) { {:sniffing => false} }
127
+
128
+ it "should not start the sniffer" do
129
+ expect(client.sniffer_thread).to be_nil
130
+ end
131
+ end
132
+ end
133
+ end
@@ -0,0 +1,58 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+ require 'stud/temporary'
3
+ require 'elasticsearch'
4
+ require "logstash/outputs/elasticsearch"
5
+
6
+ describe "Proxy option" do
7
+ let(:settings) {
8
+ {
9
+ "hosts" => "node01",
10
+ "proxy" => proxy
11
+ }
12
+ }
13
+ subject {
14
+ LogStash::Outputs::ElasticSearch.new(settings)
15
+ }
16
+
17
+ before do
18
+ allow(::Elasticsearch::Client).to receive(:new).with(any_args)
19
+ end
20
+
21
+ describe "valid configs" do
22
+ before do
23
+ subject.register
24
+ end
25
+
26
+ context "when specified as a string" do
27
+ let(:proxy) { "http://127.0.0.1:1234" }
28
+
29
+ it "should set the proxy to the exact value" do
30
+ expect(::Elasticsearch::Client).to have_received(:new) do |options|
31
+ expect(options[:transport_options][:proxy]).to eql(proxy)
32
+ end
33
+ end
34
+ end
35
+
36
+ context "when specified as a hash" do
37
+ let(:proxy) { {"hosts" => "127.0.0.1", "protocol" => "http"} }
38
+
39
+ it "should pass through the proxy values as symbols" do
40
+ expected = {:hosts => proxy["hosts"], :protocol => proxy["protocol"]}
41
+ expect(::Elasticsearch::Client).to have_received(:new) do |options|
42
+ expect(options[:transport_options][:proxy]).to eql(expected)
43
+ end
44
+ end
45
+ end
46
+ end
47
+
48
+ describe "invalid configs" do
49
+ let(:proxy) { ["bad", "stuff"] }
50
+
51
+ it "should have raised an exception" do
52
+ expect {
53
+ subject.register
54
+ }.to raise_error(LogStash::ConfigurationError)
55
+ end
56
+ end
57
+
58
+ end
@@ -0,0 +1,227 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+ require "flores/random"
3
+ require "logstash/outputs/elasticsearch"
4
+
5
+ describe "outputs/elasticsearch" do
6
+ context "with an active instance" do
7
+ let(:options) {
8
+ {
9
+ "index" => "my-index",
10
+ "hosts" => ["localhost","localhost:9202"],
11
+ "path" => "some-path"
12
+ }
13
+ }
14
+
15
+ let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
16
+
17
+ let(:manticore_host) {
18
+ eso.client.send(:client).transport.options[:hosts].first
19
+ }
20
+
21
+ around(:each) do |block|
22
+ eso.register
23
+ block.call()
24
+ eso.close
25
+ end
26
+
27
+ describe "getting a document type" do
28
+ it "should default to 'logs'" do
29
+ expect(eso.send(:get_event_type, LogStash::Event.new)).to eql("logs")
30
+ end
31
+
32
+ it "should get the type from the event if nothing else specified in the config" do
33
+ expect(eso.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
34
+ end
35
+
36
+ context "with 'document type set'" do
37
+ let(:options) { super.merge("document_type" => "bar")}
38
+ it "should get the event type from the 'document_type' setting" do
39
+ expect(eso.send(:get_event_type, LogStash::Event.new())).to eql("bar")
40
+ end
41
+ end
42
+
43
+ context "with a bad type" do
44
+ let(:type_arg) { ["foo"] }
45
+ let(:result) { eso.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
46
+
47
+ before do
48
+ allow(eso.instance_variable_get(:@logger)).to receive(:warn)
49
+ result
50
+ end
51
+
52
+ it "should call @logger.warn and return nil" do
53
+ expect(eso.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
54
+ end
55
+
56
+ it "should set the type to the stringified value" do
57
+ expect(result).to eql(type_arg.to_s)
58
+ end
59
+ end
60
+ end
61
+
62
+ describe "with path" do
63
+ it "should properly create a URI with the path" do
64
+ expect(eso.path).to eql(options["path"])
65
+ end
66
+
67
+ it "should properly set the path on the HTTP client adding slashes" do
68
+ expect(manticore_host).to include("/" + options["path"] + "/")
69
+ end
70
+
71
+ context "with extra slashes" do
72
+ let(:path) { "/slashed-path/ "}
73
+ let(:eso) {
74
+ LogStash::Outputs::ElasticSearch.new(options.merge("path" => "/some-path/"))
75
+ }
76
+
77
+ it "should properly set the path on the HTTP client without adding slashes" do
78
+ expect(manticore_host).to include(options["path"])
79
+ end
80
+ end
81
+ end
82
+ describe "without a port specified" do
83
+ it "should properly set the default port (9200) on the HTTP client" do
84
+ expect(manticore_host).to include("9200")
85
+ end
86
+ end
87
+ describe "with a port other than 9200 specified" do
88
+ let(:manticore_host) {
89
+ eso.client.send(:client).transport.options[:hosts].last
90
+ }
91
+ it "should properly set the specified port on the HTTP client" do
92
+ expect(manticore_host).to include("9202")
93
+ end
94
+ end
95
+
96
+ describe "#multi_receive" do
97
+ let(:events) { [double("one"), double("two"), double("three")] }
98
+ let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
99
+ let(:options) { super.merge("flush_size" => 2) }
100
+
101
+ before do
102
+ allow(eso).to receive(:retrying_submit).with(anything)
103
+ events.each_with_index do |e,i|
104
+ et = events_tuples[i]
105
+ allow(eso).to receive(:event_action_tuple).with(e).and_return(et)
106
+ end
107
+ eso.multi_receive(events)
108
+ end
109
+
110
+ it "should receive an array of events and invoke retrying_submit with them, split by flush_size" do
111
+ expect(eso).to have_received(:retrying_submit).with(events_tuples.slice(0,2))
112
+ expect(eso).to have_received(:retrying_submit).with(events_tuples.slice(2,3))
113
+ end
114
+
115
+ end
116
+
117
+ end
118
+
119
+ # TODO(sissel): Improve this. I'm not a fan of using message expectations (expect().to receive...)
120
+ # especially with respect to logging to verify a failure/retry has occurred. For now, this
121
+ # should suffice, though.
122
+ context "with timeout set" do
123
+ let(:listener) { Flores::Random.tcp_listener }
124
+ let(:port) { listener[2] }
125
+ let(:options) do
126
+ {
127
+ "manage_template" => false,
128
+ "hosts" => "localhost:#{port}",
129
+ "flush_size" => 1,
130
+ "timeout" => 0.1, # fast timeout
131
+ }
132
+ end
133
+ let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
134
+
135
+ before do
136
+ eso.register
137
+
138
+ # Expect a timeout to be logged.
139
+ expect(eso.logger).to receive(:error).with(/Attempted to send a bulk request/, anything)
140
+ end
141
+
142
+ after do
143
+ listener[0].close
144
+ # Stop the receive buffer, but don't flush because that would hang forever in this case since ES never returns a result
145
+ eso.instance_variable_get(:@buffer).stop(false,false)
146
+ eso.close
147
+ end
148
+
149
+ it "should fail after the timeout" do
150
+ Thread.new { eso.receive(LogStash::Event.new) }
151
+
152
+ # Allow the timeout to occur.
153
+ sleep(options["timeout"] + 0.5)
154
+ end
155
+ end
156
+
157
+ describe "the action option" do
158
+ subject(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
159
+ context "with a sprintf action" do
160
+ let(:options) { {"action" => "%{myactionfield}"} }
161
+
162
+ let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
163
+
164
+ it "should interpolate the requested action value when creating an event_action_tuple" do
165
+ expect(eso.event_action_tuple(event).first).to eql("update")
166
+ end
167
+ end
168
+
169
+ context "with an invalid action" do
170
+ let(:options) { {"action" => "SOME Garbaaage"} }
171
+
172
+ it "should raise a configuration error" do
173
+ expect { subject.register }.to raise_error(LogStash::ConfigurationError)
174
+ end
175
+ end
176
+ end
177
+
178
+ describe "SSL end to end" do
179
+ shared_examples("an encrypted client connection") do
180
+ it "should enable SSL in manticore" do
181
+ expect(eso.client.client_options[:hosts].map {|h| URI.parse(h).scheme}.uniq).to eql(['https'])
182
+ end
183
+ end
184
+
185
+ let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
186
+ subject(:manticore) { eso.client.client}
187
+
188
+ before do
189
+ eso.register
190
+ end
191
+
192
+ context "With the 'ssl' option" do
193
+ let(:options) { {"ssl" => true}}
194
+
195
+ include_examples("an encrypted client connection")
196
+ end
197
+
198
+ context "With an https host" do
199
+ let(:options) { {"hosts" => "https://localhost"} }
200
+ include_examples("an encrypted client connection")
201
+ end
202
+ end
203
+
204
+ describe "retry_on_conflict" do
205
+ let(:num_retries) { 123 }
206
+ let(:event) { LogStash::Event.new("message" => "blah") }
207
+ subject(:eso) {LogStash::Outputs::ElasticSearch.new(options.merge('retry_on_conflict' => num_retries))}
208
+
209
+ context "with a regular index" do
210
+ let(:options) { {"action" => "index"} }
211
+
212
+ it "should interpolate the requested action value when creating an event_action_tuple" do
213
+ action, params, event_data = eso.event_action_tuple(event)
214
+ expect(params).not_to include({:_retry_on_conflict => num_retries})
215
+ end
216
+ end
217
+
218
+ context "using a plain update" do
219
+ let(:options) { {"action" => "update", "retry_on_conflict" => num_retries} }
220
+
221
+ it "should interpolate the requested action value when creating an event_action_tuple" do
222
+ action, params, event_data = eso.event_action_tuple(event)
223
+ expect(params).to include({:_retry_on_conflict => num_retries})
224
+ end
225
+ end
226
+ end
227
+ end