logstash-output-elasticsearch 2.1.5-java → 2.2.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +1 -0
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +4 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +14 -5
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/integration/outputs/parent_spec.rb +73 -0
- data/spec/unit/outputs/elasticsearch/protocol_spec.rb +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0bacdce326400468748dde5d68f0d93612c481d0
|
4
|
+
data.tar.gz: ed36edd4dc675f9c5940f34314dc511134e3b0db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c1eefd941d00fcef32a5194f0bb8b457f8e8bb690cec751c4c347a89c655ea0ea4028addd7211d36e49fccda107fcbe569ed64fb5ea1869f93e0cd4738612946
|
7
|
+
data.tar.gz: 13c2da8b12fa91911e720080d75c3d0afbc984a21ce4db3b4ce150159289db63189e4455eda63005e5c170c30e59c4f6bcb5ba7611a77d465be8319f054ffb63
|
data/CHANGELOG.md
CHANGED
@@ -132,6 +132,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
132
132
|
:_routing => @routing ? event.sprintf(@routing) : nil
|
133
133
|
}
|
134
134
|
|
135
|
+
params[:parent] = event.sprintf(@parent) if @parent
|
135
136
|
params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
|
136
137
|
params
|
137
138
|
end
|
@@ -56,6 +56,10 @@ module LogStash; module Outputs; class ElasticSearch
|
|
56
56
|
# This can be dynamic using the `%{foo}` syntax.
|
57
57
|
mod.config :routing, :validate => :string
|
58
58
|
|
59
|
+
# For child documents, ID of the associated parent.
|
60
|
+
# This can be dynamic using the `%{foo}` syntax.
|
61
|
+
mod.config :parent, :validate => :string, :default => nil
|
62
|
+
|
59
63
|
# Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
|
60
64
|
# Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
|
61
65
|
# `"127.0.0.1"`
|
@@ -18,18 +18,27 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
18
18
|
# @options = DEFAULT_OPTIONS.merge(options)
|
19
19
|
@options = options
|
20
20
|
@client = build_client(@options)
|
21
|
+
# mutex to prevent requests and sniffing to access the
|
22
|
+
# connection pool at the same time
|
23
|
+
@request_mutex = Mutex.new
|
21
24
|
start_sniffing!
|
22
25
|
end
|
23
26
|
|
24
27
|
def template_install(name, template, force=false)
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
+
@request_mutex.synchronize do
|
29
|
+
if template_exists?(name) && !force
|
30
|
+
@logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
|
31
|
+
return
|
32
|
+
end
|
33
|
+
template_put(name, template)
|
28
34
|
end
|
29
|
-
template_put(name, template)
|
30
35
|
end
|
31
36
|
|
32
37
|
def bulk(actions)
|
38
|
+
@request_mutex.synchronize { non_threadsafe_bulk(actions) }
|
39
|
+
end
|
40
|
+
|
41
|
+
def non_threadsafe_bulk(actions)
|
33
42
|
return if actions.empty?
|
34
43
|
bulk_body = actions.collect do |action, args, source|
|
35
44
|
if action == 'update'
|
@@ -61,7 +70,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
61
70
|
if options[:sniffing]
|
62
71
|
@sniffer_thread = Thread.new do
|
63
72
|
loop do
|
64
|
-
sniff!
|
73
|
+
@request_mutex.synchronize { sniff! }
|
65
74
|
sleep (options[:sniffing_delay].to_f || 30)
|
66
75
|
end
|
67
76
|
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require_relative "../../../spec/es_spec_helper"
|
2
|
+
|
3
|
+
shared_examples "a parent indexer" do
|
4
|
+
let(:index) { 10.times.collect { rand(10).to_s }.join("") }
|
5
|
+
let(:type) { 10.times.collect { rand(10).to_s }.join("") }
|
6
|
+
let(:event_count) { 10000 + rand(500) }
|
7
|
+
let(:flush_size) { rand(200) + 1 }
|
8
|
+
let(:parent) { "not_implemented" }
|
9
|
+
let(:config) { "not_implemented" }
|
10
|
+
subject { LogStash::Outputs::ElasticSearch.new(config) }
|
11
|
+
|
12
|
+
before do
|
13
|
+
# Add mapping and a parent document
|
14
|
+
index_url = "http://#{get_host_port()}/#{index}"
|
15
|
+
ftw = FTW::Agent.new
|
16
|
+
mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
|
17
|
+
ftw.put!("#{index_url}", :body => mapping.to_json)
|
18
|
+
pdoc = { "foo" => "bar" }
|
19
|
+
ftw.put!("#{index_url}/#{type}_parent/test", :body => pdoc.to_json)
|
20
|
+
|
21
|
+
subject.register
|
22
|
+
event_count.times do
|
23
|
+
subject.receive(LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type))
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
it "ships events" do
|
29
|
+
index_url = "http://#{get_host_port()}/#{index}"
|
30
|
+
|
31
|
+
ftw = FTW::Agent.new
|
32
|
+
ftw.post!("#{index_url}/_refresh")
|
33
|
+
|
34
|
+
# Wait until all events are available.
|
35
|
+
Stud::try(10.times) do
|
36
|
+
query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
|
37
|
+
data = ""
|
38
|
+
response = ftw.post!("#{index_url}/_count?q=*", :body => query.to_json)
|
39
|
+
response.read_body { |chunk| data << chunk }
|
40
|
+
result = LogStash::Json.load(data)
|
41
|
+
cur_count = result["count"]
|
42
|
+
insist { cur_count } == event_count
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe "(http protocol) index events with static parent", :integration => true do
|
48
|
+
it_behaves_like 'a parent indexer' do
|
49
|
+
let(:parent) { "test" }
|
50
|
+
let(:config) {
|
51
|
+
{
|
52
|
+
"hosts" => get_host_port,
|
53
|
+
"index" => index,
|
54
|
+
"flush_size" => flush_size,
|
55
|
+
"parent" => parent
|
56
|
+
}
|
57
|
+
}
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
describe "(http_protocol) index events with fieldref in parent value", :integration => true do
|
62
|
+
it_behaves_like 'a parent indexer' do
|
63
|
+
let(:config) {
|
64
|
+
{
|
65
|
+
"hosts" => get_host_port,
|
66
|
+
"index" => index,
|
67
|
+
"flush_size" => flush_size,
|
68
|
+
"parent" => "%{link_to}"
|
69
|
+
}
|
70
|
+
}
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
@@ -25,7 +25,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
25
25
|
|
26
26
|
it "should periodically sniff the client" do
|
27
27
|
sleep 2
|
28
|
-
expect(transport).to have_received(:reload_connections!)
|
28
|
+
expect(transport).to have_received(:reload_connections!).at_least(:once)
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
@@ -206,6 +206,7 @@ files:
|
|
206
206
|
- spec/es_spec_helper.rb
|
207
207
|
- spec/integration/outputs/create_spec.rb
|
208
208
|
- spec/integration/outputs/index_spec.rb
|
209
|
+
- spec/integration/outputs/parent_spec.rb
|
209
210
|
- spec/integration/outputs/retry_spec.rb
|
210
211
|
- spec/integration/outputs/routing_spec.rb
|
211
212
|
- spec/integration/outputs/secure_spec.rb
|
@@ -247,6 +248,7 @@ test_files:
|
|
247
248
|
- spec/es_spec_helper.rb
|
248
249
|
- spec/integration/outputs/create_spec.rb
|
249
250
|
- spec/integration/outputs/index_spec.rb
|
251
|
+
- spec/integration/outputs/parent_spec.rb
|
250
252
|
- spec/integration/outputs/retry_spec.rb
|
251
253
|
- spec/integration/outputs/routing_spec.rb
|
252
254
|
- spec/integration/outputs/secure_spec.rb
|