logstash-output-elasticsearch 2.3.2-java → 2.4.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +4 -1
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +18 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +35 -15
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +15 -2
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/es_spec_helper.rb +9 -2
- data/spec/fixtures/scripts/scripted_update.groovy +2 -0
- data/spec/fixtures/scripts/scripted_update_nested.groovy +2 -0
- data/spec/fixtures/scripts/scripted_upsert.groovy +2 -0
- data/spec/integration/outputs/update_spec.rb +93 -28
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: addba3dc004cd06ac2c27a913a8339d9f6c3568b
|
4
|
+
data.tar.gz: ad5076aabc199484fa42d8f95107e22df94b661f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ecddcf3de945eff27ad89305bc12b9b419b41c31758640edde275da7e1d86881b028b26496b0595717085c149d13517d527416ea9ef05a0249e3bc5051feb46e
|
7
|
+
data.tar.gz: cf07a726ed0761fe6b8adefe598e0519c1a85f3b201043e0bb3a9a0cd123be9148717523dfb3e071e69abb660241664809f4433e73751a5a0ffd9e22cf370211
|
data/CHANGELOG.md
CHANGED
@@ -132,7 +132,10 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
132
132
|
}
|
133
133
|
|
134
134
|
params[:parent] = event.sprintf(@parent) if @parent
|
135
|
-
|
135
|
+
if @action == 'update'
|
136
|
+
params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
|
137
|
+
params[:_script] = event.sprintf(@script) if @script != ""
|
138
|
+
end
|
136
139
|
params
|
137
140
|
end
|
138
141
|
|
@@ -105,6 +105,24 @@ module LogStash; module Outputs; class ElasticSearch
|
|
105
105
|
# DEPRECATED This setting no longer does anything. It will be marked obsolete in a future version.
|
106
106
|
mod.config :max_retries, :validate => :number, :default => 3
|
107
107
|
|
108
|
+
# Set script name for scripted update mode
|
109
|
+
mod.config :script, :validate => :string, :default => ""
|
110
|
+
|
111
|
+
# Define the type of script referenced by "script" variable
|
112
|
+
# inline : "script" contains inline script
|
113
|
+
# indexed : "script" contains the name of script directly indexed in elasticsearch
|
114
|
+
# file : "script" contains the name of script stored in elasticseach's config directory
|
115
|
+
mod.config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"]
|
116
|
+
|
117
|
+
# Set the language of the used script
|
118
|
+
mod.config :script_lang, :validate => :string, :default => ""
|
119
|
+
|
120
|
+
# Set variable name passed to script (scripted update)
|
121
|
+
mod.config :script_var_name, :validate => :string, :default => "event"
|
122
|
+
|
123
|
+
# if enabled, script is in charge of creating non-existent document (scripted update)
|
124
|
+
mod.config :scripted_upsert, :validate => :boolean, :default => false
|
125
|
+
|
108
126
|
# Set max interval between bulk retries.
|
109
127
|
mod.config :retry_max_interval, :validate => :number, :default => 2
|
110
128
|
|
@@ -41,20 +41,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
41
41
|
def non_threadsafe_bulk(actions)
|
42
42
|
return if actions.empty?
|
43
43
|
bulk_body = actions.collect do |action, args, source|
|
44
|
-
if action == 'update'
|
45
|
-
if args[:_id]
|
46
|
-
source = { 'doc' => source }
|
47
|
-
if @options[:doc_as_upsert]
|
48
|
-
source['doc_as_upsert'] = true
|
49
|
-
else
|
50
|
-
source['upsert'] = args[:_upsert] if args[:_upsert]
|
51
|
-
end
|
52
|
-
else
|
53
|
-
raise(LogStash::ConfigurationError, "Specifying action => 'update' without a document '_id' is not supported.")
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
args.delete(:_upsert)
|
44
|
+
args, source = update_action_builder(args, source) if action == 'update'
|
58
45
|
|
59
46
|
if source && action != 'delete'
|
60
47
|
next [ { action => args }, source ]
|
@@ -159,5 +146,38 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
159
146
|
def template_put(name, template)
|
160
147
|
@client.indices.put_template(:name => name, :body => template)
|
161
148
|
end
|
149
|
+
|
150
|
+
# Build a bulk item for an elasticsearch update action
|
151
|
+
def update_action_builder(args, source)
|
152
|
+
if args[:_script]
|
153
|
+
# Use the event as a hash from your script with variable name defined
|
154
|
+
# by script_var_name (default: "event")
|
155
|
+
# Ex: event["@timestamp"]
|
156
|
+
source = { 'script' => {'params' => { @options[:script_var_name] => source }} }
|
157
|
+
if @options[:scripted_upsert]
|
158
|
+
source['scripted_upsert'] = true
|
159
|
+
source['upsert'] = {}
|
160
|
+
else
|
161
|
+
source['upsert'] = args.delete(:_upsert) if args[:_upsert]
|
162
|
+
end
|
163
|
+
case @options[:script_type]
|
164
|
+
when 'indexed'
|
165
|
+
source['script']['id'] = args.delete(:_script)
|
166
|
+
when 'file'
|
167
|
+
source['script']['file'] = args.delete(:_script)
|
168
|
+
when 'inline'
|
169
|
+
source['script']['inline'] = args.delete(:_script)
|
170
|
+
end
|
171
|
+
source['script']['lang'] = @options[:script_lang] if @options[:script_lang] != ''
|
172
|
+
else
|
173
|
+
source = { 'doc' => source }
|
174
|
+
if @options[:doc_as_upsert]
|
175
|
+
source['doc_as_upsert'] = true
|
176
|
+
else
|
177
|
+
source['upsert'] = args.delete(:_upsert) if args[:_upsert]
|
178
|
+
end
|
179
|
+
end
|
180
|
+
[args, source]
|
181
|
+
end
|
162
182
|
end
|
163
|
-
end end end
|
183
|
+
end end end
|
@@ -17,10 +17,23 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
17
17
|
client_settings.merge! setup_proxy(logger, params)
|
18
18
|
common_options.merge! setup_basic_auth(logger, params)
|
19
19
|
|
20
|
+
# Update API setup
|
21
|
+
raise( Logstash::ConfigurationError,
|
22
|
+
"doc_as_upsert and scripted_upsert are mutually exclusive."
|
23
|
+
) if params["doc_as_upsert"] and params["scripted_upsert"]
|
24
|
+
|
25
|
+
raise(
|
26
|
+
LogStash::ConfigurationError,
|
27
|
+
"Specifying action => 'update' needs a document_id."
|
28
|
+
) if params['action'] == 'update' and params.fetch('document_id', '') == ''
|
29
|
+
|
20
30
|
# Update API setup
|
21
31
|
update_options = {
|
22
|
-
:
|
23
|
-
:
|
32
|
+
:doc_as_upsert => params["doc_as_upsert"],
|
33
|
+
:script_var_name => params["script_var_name"],
|
34
|
+
:script_type => params["script_type"],
|
35
|
+
:script_lang => params["script_lang"],
|
36
|
+
:scripted_upsert => params["scripted_upsert"]
|
24
37
|
}
|
25
38
|
common_options.merge! update_options if params["action"] == 'update'
|
26
39
|
|
data/spec/es_spec_helper.rb
CHANGED
@@ -8,7 +8,7 @@ require "logstash/outputs/elasticsearch"
|
|
8
8
|
|
9
9
|
CONTAINER_NAME = "logstash-output-elasticsearch-#{rand(999).to_s}"
|
10
10
|
CONTAINER_IMAGE = "elasticsearch"
|
11
|
-
CONTAINER_TAG = "
|
11
|
+
CONTAINER_TAG = "2.0"
|
12
12
|
|
13
13
|
DOCKER_INTEGRATION = ENV["DOCKER_INTEGRATION"]
|
14
14
|
|
@@ -45,7 +45,14 @@ RSpec.configure do |config|
|
|
45
45
|
ls = Longshoreman::new
|
46
46
|
ls.container.get(CONTAINER_NAME)
|
47
47
|
rescue Docker::Error::NotFoundError
|
48
|
-
|
48
|
+
scriptDir = File.expand_path File.dirname(__FILE__) + '/fixtures/scripts'
|
49
|
+
Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME, {
|
50
|
+
'Cmd' => [ "-Des.script.inline=on", "-Des.script.indexed=on" ],
|
51
|
+
'HostConfig' => {
|
52
|
+
'Binds' => ["#{scriptDir}:/usr/share/elasticsearch/config/scripts"],
|
53
|
+
'PublishAllPorts' => true
|
54
|
+
}
|
55
|
+
})
|
49
56
|
# TODO(talevy): verify ES is running instead of static timeout
|
50
57
|
sleep 10
|
51
58
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
|
3
|
-
describe "
|
3
|
+
describe "Update actions", :integration => true do
|
4
4
|
require "logstash/outputs/elasticsearch"
|
5
5
|
require "elasticsearch"
|
6
6
|
|
7
|
-
def get_es_output(
|
7
|
+
def get_es_output( options={} )
|
8
8
|
settings = {
|
9
9
|
"manage_template" => true,
|
10
10
|
"index" => "logstash-update",
|
@@ -12,10 +12,7 @@ describe "all protocols update actions", :integration => true do
|
|
12
12
|
"hosts" => get_host_port(),
|
13
13
|
"action" => "update"
|
14
14
|
}
|
15
|
-
settings
|
16
|
-
settings['document_id'] = id unless id.nil?
|
17
|
-
settings['doc_as_upsert'] = doc_as_upsert unless doc_as_upsert.nil?
|
18
|
-
LogStash::Outputs::ElasticSearch.new(settings)
|
15
|
+
LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
|
19
16
|
end
|
20
17
|
|
21
18
|
before :each do
|
@@ -29,39 +26,87 @@ describe "all protocols update actions", :integration => true do
|
|
29
26
|
:index => 'logstash-update',
|
30
27
|
:type => 'logs',
|
31
28
|
:id => "123",
|
32
|
-
:body => { :message => 'Test' }
|
29
|
+
:body => { :message => 'Test', :counter => 1 }
|
33
30
|
)
|
34
31
|
@es.indices.refresh
|
35
32
|
end
|
36
33
|
|
37
34
|
it "should fail without a document_id" do
|
38
|
-
event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
|
39
|
-
action = ["update", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
|
40
35
|
subject = get_es_output
|
41
|
-
subject.register
|
42
|
-
expect { subject.flush([action]) }.to raise_error
|
36
|
+
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
43
37
|
end
|
44
38
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
39
|
+
context "when update only" do
|
40
|
+
it "should not create new document" do
|
41
|
+
subject = get_es_output({ 'document_id' => "456" } )
|
42
|
+
subject.register
|
43
|
+
subject.receive(LogStash::Event.new("message" => "sample message here"))
|
44
|
+
subject.flush
|
45
|
+
expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should update existing document" do
|
49
|
+
subject = get_es_output({ 'document_id' => "123" })
|
50
|
+
subject.register
|
51
|
+
subject.receive(LogStash::Event.new("message" => "updated message here"))
|
52
|
+
subject.flush
|
53
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
54
|
+
insist { r["_source"]["message"] } == 'updated message here'
|
55
|
+
end
|
51
56
|
end
|
57
|
+
|
58
|
+
context "when using script" do
|
59
|
+
it "should increment a counter with event/doc 'count' variable" do
|
60
|
+
subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
|
61
|
+
subject.register
|
62
|
+
subject.receive(LogStash::Event.new("count" => 2))
|
63
|
+
subject.flush
|
64
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
65
|
+
insist { r["_source"]["counter"] } == 3
|
66
|
+
end
|
52
67
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
68
|
+
it "should increment a counter with event/doc '[data][count]' nested variable" do
|
69
|
+
subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
|
70
|
+
subject.register
|
71
|
+
subject.receive(LogStash::Event.new("data" => { "count" => 3 }))
|
72
|
+
subject.flush
|
73
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
74
|
+
insist { r["_source"]["counter"] } == 4
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should increment a counter with event/doc 'count' variable with inline script" do
|
78
|
+
subject = get_es_output({
|
79
|
+
'document_id' => "123",
|
80
|
+
'script' => 'ctx._source.counter += event["count"]',
|
81
|
+
'script_lang' => 'groovy',
|
82
|
+
'script_type' => 'inline'
|
83
|
+
})
|
84
|
+
subject.register
|
85
|
+
subject.receive(LogStash::Event.new("count" => 3 ))
|
86
|
+
subject.flush
|
87
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
88
|
+
insist { r["_source"]["counter"] } == 4
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should increment a counter with event/doc 'count' variable with indexed script" do
|
92
|
+
@es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
|
93
|
+
subject = get_es_output({
|
94
|
+
'document_id' => "123",
|
95
|
+
'script' => 'indexed_update',
|
96
|
+
'script_lang' => 'groovy',
|
97
|
+
'script_type' => 'indexed'
|
98
|
+
})
|
99
|
+
subject.register
|
100
|
+
subject.receive(LogStash::Event.new("count" => 4 ))
|
101
|
+
subject.flush
|
102
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
103
|
+
insist { r["_source"]["counter"] } == 5
|
104
|
+
end
|
60
105
|
end
|
61
106
|
|
62
|
-
context "
|
63
|
-
it "should create new documents with upsert
|
64
|
-
subject = get_es_output("456", '{"message": "upsert message"}')
|
107
|
+
context "when update with upsert" do
|
108
|
+
it "should create new documents with provided upsert" do
|
109
|
+
subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
65
110
|
subject.register
|
66
111
|
subject.receive(LogStash::Event.new("message" => "sample message here"))
|
67
112
|
subject.flush
|
@@ -70,12 +115,32 @@ describe "all protocols update actions", :integration => true do
|
|
70
115
|
end
|
71
116
|
|
72
117
|
it "should create new documents with event/doc as upsert" do
|
73
|
-
subject = get_es_output("456",
|
118
|
+
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
|
74
119
|
subject.register
|
75
120
|
subject.receive(LogStash::Event.new("message" => "sample message here"))
|
76
121
|
subject.flush
|
77
122
|
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
78
123
|
insist { r["_source"]["message"] } == 'sample message here'
|
79
124
|
end
|
125
|
+
|
126
|
+
context "when using script" do
|
127
|
+
it "should create new documents with upsert content" do
|
128
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
|
129
|
+
subject.register
|
130
|
+
subject.receive(LogStash::Event.new("message" => "sample message here"))
|
131
|
+
subject.flush
|
132
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
133
|
+
insist { r["_source"]["message"] } == 'upsert message'
|
134
|
+
end
|
135
|
+
|
136
|
+
it "should create new documents with event/doc as script params" do
|
137
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
|
138
|
+
subject.register
|
139
|
+
subject.receive(LogStash::Event.new("counter" => 1))
|
140
|
+
subject.flush
|
141
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
142
|
+
insist { r["_source"]["counter"] } == 1
|
143
|
+
end
|
144
|
+
end
|
80
145
|
end
|
81
146
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.4.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -210,6 +210,9 @@ files:
|
|
210
210
|
- lib/logstash/outputs/elasticsearch/template_manager.rb
|
211
211
|
- logstash-output-elasticsearch.gemspec
|
212
212
|
- spec/es_spec_helper.rb
|
213
|
+
- spec/fixtures/scripts/scripted_update.groovy
|
214
|
+
- spec/fixtures/scripts/scripted_update_nested.groovy
|
215
|
+
- spec/fixtures/scripts/scripted_upsert.groovy
|
213
216
|
- spec/integration/outputs/create_spec.rb
|
214
217
|
- spec/integration/outputs/index_spec.rb
|
215
218
|
- spec/integration/outputs/parent_spec.rb
|
@@ -252,6 +255,9 @@ specification_version: 4
|
|
252
255
|
summary: Logstash Output to Elasticsearch
|
253
256
|
test_files:
|
254
257
|
- spec/es_spec_helper.rb
|
258
|
+
- spec/fixtures/scripts/scripted_update.groovy
|
259
|
+
- spec/fixtures/scripts/scripted_update_nested.groovy
|
260
|
+
- spec/fixtures/scripts/scripted_upsert.groovy
|
255
261
|
- spec/integration/outputs/create_spec.rb
|
256
262
|
- spec/integration/outputs/index_spec.rb
|
257
263
|
- spec/integration/outputs/parent_spec.rb
|