logstash-output-elasticsearch 2.1.2-java → 2.1.4-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,8 +9,7 @@ describe "all protocols update actions", :integration => true do
9
9
  "manage_template" => true,
10
10
  "index" => "logstash-update",
11
11
  "template_overwrite" => true,
12
- "hosts" => get_host(),
13
- "port" => get_port(),
12
+ "hosts" => get_host_port(),
14
13
  "action" => "update"
15
14
  }
16
15
  settings['upsert'] = upsert unless upsert.nil?
@@ -47,7 +46,7 @@ describe "all protocols update actions", :integration => true do
47
46
  subject = get_es_output("456")
48
47
  subject.register
49
48
  subject.receive(LogStash::Event.new("message" => "sample message here"))
50
- subject.buffer_flush(:final => true)
49
+ subject.flush
51
50
  expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
52
51
  end
53
52
 
@@ -55,7 +54,7 @@ describe "all protocols update actions", :integration => true do
55
54
  subject = get_es_output("123")
56
55
  subject.register
57
56
  subject.receive(LogStash::Event.new("message" => "updated message here"))
58
- subject.buffer_flush(:final => true)
57
+ subject.flush
59
58
  r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
60
59
  insist { r["_source"]["message"] } == 'updated message here'
61
60
  end
@@ -65,7 +64,7 @@ describe "all protocols update actions", :integration => true do
65
64
  subject = get_es_output("456", '{"message": "upsert message"}')
66
65
  subject.register
67
66
  subject.receive(LogStash::Event.new("message" => "sample message here"))
68
- subject.buffer_flush(:final => true)
67
+ subject.flush
69
68
  r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
70
69
  insist { r["_source"]["message"] } == 'upsert message'
71
70
  end
@@ -74,7 +73,7 @@ describe "all protocols update actions", :integration => true do
74
73
  subject = get_es_output("456", nil, true)
75
74
  subject.register
76
75
  subject.receive(LogStash::Event.new("message" => "sample message here"))
77
- subject.buffer_flush(:final => true)
76
+ subject.flush
78
77
  r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
79
78
  insist { r["_source"]["message"] } == 'sample message here'
80
79
  end
@@ -0,0 +1,118 @@
1
+ require "logstash/outputs/elasticsearch"
2
+ require "logstash/outputs/elasticsearch/buffer"
3
+
4
+ describe LogStash::Outputs::ElasticSearch::Buffer do
5
+ class OperationTarget # Used to track buffer flushesn
6
+ attr_reader :buffer, :buffer_history, :receive_count
7
+ def initialize
8
+ @buffer = nil
9
+ @buffer_history = []
10
+ @receive_count = 0
11
+ end
12
+
13
+ def receive(buffer)
14
+ @receive_count += 1
15
+ @buffer_history << buffer.clone
16
+ @buffer = buffer
17
+ end
18
+ end
19
+
20
+ let(:logger) { Cabin::Channel.get }
21
+ let(:max_size) { 10 }
22
+ let(:flush_interval) { 2 }
23
+ # Used to track flush count
24
+ let(:operation_target) { OperationTarget.new() }
25
+ let(:operation) { proc {|buffer| operation_target.receive(buffer) } }
26
+ subject(:buffer){ LogStash::Outputs::ElasticSearch::Buffer.new(logger, max_size, flush_interval, &operation) }
27
+
28
+ after(:each) do
29
+ buffer.stop(do_flush=false)
30
+ end
31
+
32
+ it "should initialize cleanly" do
33
+ expect(buffer).to be_a(LogStash::Outputs::ElasticSearch::Buffer)
34
+ end
35
+
36
+ shared_examples("a buffer with two items inside") do
37
+ it "should add a pushed item to the buffer" do
38
+ buffer.synchronize do |data|
39
+ expect(data).to include(item1)
40
+ expect(data).to include(item2)
41
+ end
42
+ end
43
+
44
+ describe "interval flushing" do
45
+ before do
46
+ sleep flush_interval + 1
47
+ end
48
+
49
+ it "should flush the buffer after the interval has passed" do
50
+ expect(operation_target.receive_count).to eql(1)
51
+ end
52
+
53
+ it "should clear the buffer after a successful flush" do
54
+ expect(operation_target.buffer).to eql([])
55
+ end
56
+ end
57
+
58
+ describe "interval flushing a stopped buffer" do
59
+ before do
60
+ buffer.stop(do_flush=false)
61
+ sleep flush_interval + 1
62
+ end
63
+
64
+ it "should not flush if the buffer is stopped" do
65
+ expect(operation_target.receive_count).to eql(0)
66
+ end
67
+ end
68
+ end
69
+
70
+ describe "with a buffer push" do
71
+ let(:item1) { "foo" }
72
+ let(:item2) { "bar" }
73
+
74
+ describe "a buffer with two items pushed to it separately" do
75
+ before do
76
+ buffer << item1
77
+ buffer << item2
78
+ end
79
+
80
+ include_examples("a buffer with two items inside")
81
+ end
82
+
83
+ describe "a buffer with two items pushed to it in one operation" do
84
+ before do
85
+ buffer.push_multi([item1, item2])
86
+ end
87
+
88
+ include_examples("a buffer with two items inside")
89
+ end
90
+ end
91
+
92
+ describe "with an empty buffer" do
93
+ it "should not perform an operation if the buffer is empty" do
94
+ buffer.flush
95
+ expect(operation_target.receive_count).to eql(0)
96
+ end
97
+ end
98
+
99
+ describe "flushing with an operation that raises an error" do
100
+ class TestError < StandardError; end
101
+ let(:operation) { proc {|buffer| raise TestError, "A test" } }
102
+ let(:item) { double("item") }
103
+
104
+ before do
105
+ buffer << item
106
+ end
107
+
108
+ it "should raise an exception" do
109
+ expect { buffer.flush }.to raise_error(TestError)
110
+ end
111
+
112
+ it "should not clear the buffer" do
113
+ expect do
114
+ buffer.flush rescue TestError
115
+ end.not_to change(buffer, :contents)
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,27 @@
1
+ require "logstash/outputs/elasticsearch"
2
+ require "logstash/outputs/elasticsearch/http_client"
3
+ require "logstash/outputs/elasticsearch/http_client_builder"
4
+
5
+ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
6
+ describe "auth setup with url encodable passwords" do
7
+ let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
8
+ let(:user) { "foo@bar"}
9
+ let(:password) {"baz@blah" }
10
+ let(:password_secured) do
11
+ secured = double("password")
12
+ allow(secured).to receive(:value).and_return(password)
13
+ secured
14
+ end
15
+ let(:options) { {"user" => user, "password" => password} }
16
+ let(:logger) { mock("logger") }
17
+ let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
18
+
19
+ it "should return the user verbatim" do
20
+ expect(auth_setup[:user]).to eql(user)
21
+ end
22
+
23
+ it "should return the password verbatim" do
24
+ expect(auth_setup[:password]).to eql(password)
25
+ end
26
+ end
27
+ end
@@ -2,10 +2,10 @@ require "logstash/devutils/rspec/spec_helper"
2
2
  require "logstash/outputs/elasticsearch/http_client"
3
3
  require "java"
4
4
 
5
- describe LogStash::Outputs::Elasticsearch::HttpClient do
5
+ describe LogStash::Outputs::ElasticSearch::HttpClient do
6
6
  describe "sniffing" do
7
7
  let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get }}
8
- let(:client) { LogStash::Outputs::Elasticsearch::HttpClient.new(base_options.merge(client_opts)) }
8
+ let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
9
9
  let(:transport) { client.client.transport }
10
10
 
11
11
  before do
@@ -1,11 +1,9 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
  require "flores/random"
3
+ require "logstash/outputs/elasticsearch"
3
4
 
4
5
  describe "outputs/elasticsearch" do
5
- describe "http client create" do
6
- require "logstash/outputs/elasticsearch"
7
- require "elasticsearch"
8
-
6
+ context "with an active instance" do
9
7
  let(:options) {
10
8
  {
11
9
  "index" => "my-index",
@@ -21,9 +19,44 @@ describe "outputs/elasticsearch" do
21
19
  }
22
20
 
23
21
  around(:each) do |block|
24
- thread = eso.register
22
+ eso.register
25
23
  block.call()
26
- thread.kill()
24
+ eso.close
25
+ end
26
+
27
+ describe "getting a document type" do
28
+ it "should default to 'logs'" do
29
+ expect(eso.send(:get_event_type, LogStash::Event.new)).to eql("logs")
30
+ end
31
+
32
+ it "should get the type from the event if nothing else specified in the config" do
33
+ expect(eso.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
34
+ end
35
+
36
+ context "with 'document type set'" do
37
+ let(:options) { super.merge("document_type" => "bar")}
38
+ it "should get the event type from the 'document_type' setting" do
39
+ expect(eso.send(:get_event_type, LogStash::Event.new())).to eql("bar")
40
+ end
41
+ end
42
+
43
+ context "with a bad type" do
44
+ let(:type_arg) { ["foo"] }
45
+ let(:result) { eso.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
46
+
47
+ before do
48
+ allow(eso.instance_variable_get(:@logger)).to receive(:warn)
49
+ result
50
+ end
51
+
52
+ it "should call @logger.warn and return nil" do
53
+ expect(eso.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
54
+ end
55
+
56
+ it "should set the type to the stringified value" do
57
+ expect(result).to eql(type_arg.to_s)
58
+ end
59
+ end
27
60
  end
28
61
 
29
62
  describe "with path" do
@@ -62,7 +95,7 @@ describe "outputs/elasticsearch" do
62
95
  end
63
96
 
64
97
  # TODO(sissel): Improve this. I'm not a fan of using message expectations (expect().to receive...)
65
- # especially with respect to logging to verify a failure/retry has occurred. For now, this
98
+ # especially with respect to logging to verify a failure/retry has occurred. For now, this
66
99
  # should suffice, though.
67
100
  context "with timeout set" do
68
101
  let(:listener) { Flores::Random.tcp_listener }
@@ -76,17 +109,18 @@ describe "outputs/elasticsearch" do
76
109
  }
77
110
  end
78
111
  let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
79
-
112
+
80
113
  before do
81
- eso.register
114
+ eso.register
82
115
 
83
116
  # Expect a timeout to be logged.
84
- expect(eso.logger).to receive(:warn).with("Failed to flush outgoing items",
85
- hash_including(:exception => "Manticore::SocketTimeout"))
117
+ expect(eso.logger).to receive(:error).with(/Attempted to send a bulk request/, anything)
86
118
  end
87
119
 
88
120
  after do
89
121
  listener[0].close
122
+ # Stop the receive buffer, but don't flush because that would hang forever in this case since ES never returns a result
123
+ eso.instance_variable_get(:@buffer).stop(false,false)
90
124
  eso.close
91
125
  end
92
126
 
@@ -94,7 +128,7 @@ describe "outputs/elasticsearch" do
94
128
  Thread.new { eso.receive(LogStash::Event.new) }
95
129
 
96
130
  # Allow the timeout to occur.
97
- sleep(options["timeout"] + 0.3)
131
+ sleep(options["timeout"] + 0.5)
98
132
  end
99
133
  end
100
134
  end
@@ -27,7 +27,6 @@ describe "SSL option" do
27
27
  end
28
28
 
29
29
  context "when using ssl with client certificates" do
30
-
31
30
  let(:keystore_path) { Stud::Temporary.file.path }
32
31
 
33
32
  after :each do
metadata CHANGED
@@ -1,39 +1,30 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.2
4
+ version: 2.1.4
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-22 00:00:00.000000000 Z
11
+ date: 2015-11-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: concurrent-ruby
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - '>='
18
- - !ruby/object:Gem::Version
19
- version: '0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - '>='
23
17
  - !ruby/object:Gem::Version
24
18
  version: '0'
19
+ name: concurrent-ruby
25
20
  prerelease: false
26
21
  type: :runtime
27
- - !ruby/object:Gem::Dependency
28
- name: elasticsearch
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
24
  - - '>='
32
25
  - !ruby/object:Gem::Version
33
- version: 1.0.13
34
- - - ~>
35
- - !ruby/object:Gem::Version
36
- version: '1.0'
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
37
28
  requirement: !ruby/object:Gem::Requirement
38
29
  requirements:
39
30
  - - '>='
@@ -42,18 +33,18 @@ dependencies:
42
33
  - - ~>
43
34
  - !ruby/object:Gem::Version
44
35
  version: '1.0'
36
+ name: elasticsearch
45
37
  prerelease: false
46
38
  type: :runtime
47
- - !ruby/object:Gem::Dependency
48
- name: stud
49
39
  version_requirements: !ruby/object:Gem::Requirement
50
40
  requirements:
51
41
  - - '>='
52
42
  - !ruby/object:Gem::Version
53
- version: 0.0.17
43
+ version: 1.0.13
54
44
  - - ~>
55
45
  - !ruby/object:Gem::Version
56
- version: '0.0'
46
+ version: '1.0'
47
+ - !ruby/object:Gem::Dependency
57
48
  requirement: !ruby/object:Gem::Requirement
58
49
  requirements:
59
50
  - - '>='
@@ -62,135 +53,156 @@ dependencies:
62
53
  - - ~>
63
54
  - !ruby/object:Gem::Version
64
55
  version: '0.0'
56
+ name: stud
65
57
  prerelease: false
66
58
  type: :runtime
67
- - !ruby/object:Gem::Dependency
68
- name: cabin
69
59
  version_requirements: !ruby/object:Gem::Requirement
70
60
  requirements:
61
+ - - '>='
62
+ - !ruby/object:Gem::Version
63
+ version: 0.0.17
71
64
  - - ~>
72
65
  - !ruby/object:Gem::Version
73
- version: '0.6'
66
+ version: '0.0'
67
+ - !ruby/object:Gem::Dependency
74
68
  requirement: !ruby/object:Gem::Requirement
75
69
  requirements:
76
70
  - - ~>
77
71
  - !ruby/object:Gem::Version
78
72
  version: '0.6'
73
+ name: cabin
79
74
  prerelease: false
80
75
  type: :runtime
81
- - !ruby/object:Gem::Dependency
82
- name: logstash-core
83
76
  version_requirements: !ruby/object:Gem::Requirement
84
77
  requirements:
85
- - - '>='
86
- - !ruby/object:Gem::Version
87
- version: 2.0.0.beta2
88
- - - <
78
+ - - ~>
89
79
  - !ruby/object:Gem::Version
90
- version: 3.0.0
80
+ version: '0.6'
81
+ - !ruby/object:Gem::Dependency
91
82
  requirement: !ruby/object:Gem::Requirement
92
83
  requirements:
93
84
  - - '>='
94
85
  - !ruby/object:Gem::Version
95
- version: 2.0.0.beta2
86
+ version: 2.0.0
96
87
  - - <
97
88
  - !ruby/object:Gem::Version
98
89
  version: 3.0.0
90
+ name: logstash-core
99
91
  prerelease: false
100
92
  type: :runtime
101
- - !ruby/object:Gem::Dependency
102
- name: ftw
103
93
  version_requirements: !ruby/object:Gem::Requirement
104
94
  requirements:
105
- - - ~>
95
+ - - '>='
106
96
  - !ruby/object:Gem::Version
107
- version: 0.0.42
97
+ version: 2.0.0
98
+ - - <
99
+ - !ruby/object:Gem::Version
100
+ version: 3.0.0
101
+ - !ruby/object:Gem::Dependency
108
102
  requirement: !ruby/object:Gem::Requirement
109
103
  requirements:
110
104
  - - ~>
111
105
  - !ruby/object:Gem::Version
112
106
  version: 0.0.42
107
+ name: ftw
113
108
  prerelease: false
114
109
  type: :development
115
- - !ruby/object:Gem::Dependency
116
- name: logstash-codec-plain
117
110
  version_requirements: !ruby/object:Gem::Requirement
118
111
  requirements:
119
- - - '>='
112
+ - - ~>
120
113
  - !ruby/object:Gem::Version
121
- version: '0'
114
+ version: 0.0.42
115
+ - !ruby/object:Gem::Dependency
122
116
  requirement: !ruby/object:Gem::Requirement
123
117
  requirements:
124
118
  - - '>='
125
119
  - !ruby/object:Gem::Version
126
120
  version: '0'
121
+ name: logstash-codec-plain
127
122
  prerelease: false
128
123
  type: :development
129
- - !ruby/object:Gem::Dependency
130
- name: manticore
131
124
  version_requirements: !ruby/object:Gem::Requirement
132
125
  requirements:
133
- - - ~>
126
+ - - '>='
134
127
  - !ruby/object:Gem::Version
135
- version: 0.4.2
128
+ version: '0'
129
+ - !ruby/object:Gem::Dependency
136
130
  requirement: !ruby/object:Gem::Requirement
137
131
  requirements:
138
132
  - - ~>
139
133
  - !ruby/object:Gem::Version
140
134
  version: 0.4.2
135
+ name: manticore
141
136
  prerelease: false
142
137
  type: :runtime
143
- - !ruby/object:Gem::Dependency
144
- name: logstash-devutils
145
138
  version_requirements: !ruby/object:Gem::Requirement
146
139
  requirements:
147
- - - '>='
140
+ - - ~>
148
141
  - !ruby/object:Gem::Version
149
- version: '0'
142
+ version: 0.4.2
143
+ - !ruby/object:Gem::Dependency
150
144
  requirement: !ruby/object:Gem::Requirement
151
145
  requirements:
152
146
  - - '>='
153
147
  - !ruby/object:Gem::Version
154
148
  version: '0'
149
+ name: logstash-devutils
155
150
  prerelease: false
156
151
  type: :development
157
- - !ruby/object:Gem::Dependency
158
- name: longshoreman
159
152
  version_requirements: !ruby/object:Gem::Requirement
160
153
  requirements:
161
154
  - - '>='
162
155
  - !ruby/object:Gem::Version
163
156
  version: '0'
157
+ - !ruby/object:Gem::Dependency
164
158
  requirement: !ruby/object:Gem::Requirement
165
159
  requirements:
166
160
  - - '>='
167
161
  - !ruby/object:Gem::Version
168
162
  version: '0'
163
+ name: longshoreman
169
164
  prerelease: false
170
165
  type: :development
171
- - !ruby/object:Gem::Dependency
172
- name: flores
173
166
  version_requirements: !ruby/object:Gem::Requirement
174
167
  requirements:
175
168
  - - '>='
176
169
  - !ruby/object:Gem::Version
177
170
  version: '0'
171
+ - !ruby/object:Gem::Dependency
178
172
  requirement: !ruby/object:Gem::Requirement
179
173
  requirements:
180
174
  - - '>='
181
175
  - !ruby/object:Gem::Version
182
176
  version: '0'
177
+ name: flores
183
178
  prerelease: false
184
179
  type: :development
180
+ version_requirements: !ruby/object:Gem::Requirement
181
+ requirements:
182
+ - - '>='
183
+ - !ruby/object:Gem::Version
184
+ version: '0'
185
185
  description: Output events to elasticsearch
186
186
  email: info@elastic.co
187
187
  executables: []
188
188
  extensions: []
189
189
  extra_rdoc_files: []
190
190
  files:
191
+ - CHANGELOG.md
192
+ - CONTRIBUTORS
193
+ - Gemfile
194
+ - LICENSE
195
+ - NOTICE.TXT
196
+ - README.md
191
197
  - lib/logstash/outputs/elasticsearch.rb
198
+ - lib/logstash/outputs/elasticsearch/buffer.rb
199
+ - lib/logstash/outputs/elasticsearch/common.rb
200
+ - lib/logstash/outputs/elasticsearch/common_configs.rb
192
201
  - lib/logstash/outputs/elasticsearch/elasticsearch-template.json
193
202
  - lib/logstash/outputs/elasticsearch/http_client.rb
203
+ - lib/logstash/outputs/elasticsearch/http_client_builder.rb
204
+ - lib/logstash/outputs/elasticsearch/template_manager.rb
205
+ - logstash-output-elasticsearch.gemspec
194
206
  - spec/es_spec_helper.rb
195
207
  - spec/integration/outputs/create_spec.rb
196
208
  - spec/integration/outputs/index_spec.rb
@@ -199,17 +211,12 @@ files:
199
211
  - spec/integration/outputs/secure_spec.rb
200
212
  - spec/integration/outputs/templates_spec.rb
201
213
  - spec/integration/outputs/update_spec.rb
214
+ - spec/unit/buffer_spec.rb
215
+ - spec/unit/http_client_builder_spec.rb
216
+ - spec/unit/outputs/elasticsearch/protocol_spec.rb
202
217
  - spec/unit/outputs/elasticsearch_proxy_spec.rb
203
218
  - spec/unit/outputs/elasticsearch_spec.rb
204
219
  - spec/unit/outputs/elasticsearch_ssl_spec.rb
205
- - spec/unit/outputs/elasticsearch/protocol_spec.rb
206
- - logstash-output-elasticsearch.gemspec
207
- - README.md
208
- - CHANGELOG.md
209
- - CONTRIBUTORS
210
- - Gemfile
211
- - LICENSE
212
- - NOTICE.TXT
213
220
  homepage: http://logstash.net/
214
221
  licenses:
215
222
  - apache-2.0
@@ -232,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
232
239
  version: '0'
233
240
  requirements: []
234
241
  rubyforge_project:
235
- rubygems_version: 2.1.9
242
+ rubygems_version: 2.4.8
236
243
  signing_key:
237
244
  specification_version: 4
238
245
  summary: Logstash Output to Elasticsearch
@@ -245,7 +252,9 @@ test_files:
245
252
  - spec/integration/outputs/secure_spec.rb
246
253
  - spec/integration/outputs/templates_spec.rb
247
254
  - spec/integration/outputs/update_spec.rb
255
+ - spec/unit/buffer_spec.rb
256
+ - spec/unit/http_client_builder_spec.rb
257
+ - spec/unit/outputs/elasticsearch/protocol_spec.rb
248
258
  - spec/unit/outputs/elasticsearch_proxy_spec.rb
249
259
  - spec/unit/outputs/elasticsearch_spec.rb
250
260
  - spec/unit/outputs/elasticsearch_ssl_spec.rb
251
- - spec/unit/outputs/elasticsearch/protocol_spec.rb