logstash-output-elasticsearch 2.3.2-java → 2.4.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6adf3c643188045a088ba2644b722587058c4ede
4
- data.tar.gz: bcc360c9005ac58487c1b77bb34656c34f94dc07
3
+ metadata.gz: addba3dc004cd06ac2c27a913a8339d9f6c3568b
4
+ data.tar.gz: ad5076aabc199484fa42d8f95107e22df94b661f
5
5
  SHA512:
6
- metadata.gz: 6219cecf9a5087d54264f7554361f1bed8ead48f4d86f509f39ec47a421b629d4ecbadd2df171ac39bedbaae6c8d5f7f0b46183e920a96bf8ffdbc5850762b3a
7
- data.tar.gz: 5159a07fcc22e82f4bc6c711f51d95e46cdca302758bb6f51e19dc1897ba4c2df2243bba3b6adcd7372e41c371cbaf7c3f5cade6d99a15d21e342b9b5cca1377
6
+ metadata.gz: ecddcf3de945eff27ad89305bc12b9b419b41c31758640edde275da7e1d86881b028b26496b0595717085c149d13517d527416ea9ef05a0249e3bc5051feb46e
7
+ data.tar.gz: cf07a726ed0761fe6b8adefe598e0519c1a85f3b201043e0bb3a9a0cd123be9148717523dfb3e071e69abb660241664809f4433e73751a5a0ffd9e22cf370211
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 2.4.0
2
+ - Scripted update support courtesy of @Da-Wei
3
+
1
4
  ## 2.3.2
2
5
  - Fix bug where max_retry_interval was not respected for HTTP error codes
3
6
 
@@ -132,7 +132,10 @@ module LogStash; module Outputs; class ElasticSearch;
132
132
  }
133
133
 
134
134
  params[:parent] = event.sprintf(@parent) if @parent
135
- params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
135
+ if @action == 'update'
136
+ params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
137
+ params[:_script] = event.sprintf(@script) if @script != ""
138
+ end
136
139
  params
137
140
  end
138
141
 
@@ -105,6 +105,24 @@ module LogStash; module Outputs; class ElasticSearch
105
105
  # DEPRECATED This setting no longer does anything. It will be marked obsolete in a future version.
106
106
  mod.config :max_retries, :validate => :number, :default => 3
107
107
 
108
+ # Set script name for scripted update mode
109
+ mod.config :script, :validate => :string, :default => ""
110
+
111
+ # Define the type of script referenced by "script" variable
112
+ # inline : "script" contains inline script
113
+ # indexed : "script" contains the name of script directly indexed in elasticsearch
114
+ # file : "script" contains the name of script stored in elasticseach's config directory
115
+ mod.config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"]
116
+
117
+ # Set the language of the used script
118
+ mod.config :script_lang, :validate => :string, :default => ""
119
+
120
+ # Set variable name passed to script (scripted update)
121
+ mod.config :script_var_name, :validate => :string, :default => "event"
122
+
123
+ # if enabled, script is in charge of creating non-existent document (scripted update)
124
+ mod.config :scripted_upsert, :validate => :boolean, :default => false
125
+
108
126
  # Set max interval between bulk retries.
109
127
  mod.config :retry_max_interval, :validate => :number, :default => 2
110
128
 
@@ -41,20 +41,7 @@ module LogStash; module Outputs; class ElasticSearch;
41
41
  def non_threadsafe_bulk(actions)
42
42
  return if actions.empty?
43
43
  bulk_body = actions.collect do |action, args, source|
44
- if action == 'update'
45
- if args[:_id]
46
- source = { 'doc' => source }
47
- if @options[:doc_as_upsert]
48
- source['doc_as_upsert'] = true
49
- else
50
- source['upsert'] = args[:_upsert] if args[:_upsert]
51
- end
52
- else
53
- raise(LogStash::ConfigurationError, "Specifying action => 'update' without a document '_id' is not supported.")
54
- end
55
- end
56
-
57
- args.delete(:_upsert)
44
+ args, source = update_action_builder(args, source) if action == 'update'
58
45
 
59
46
  if source && action != 'delete'
60
47
  next [ { action => args }, source ]
@@ -159,5 +146,38 @@ module LogStash; module Outputs; class ElasticSearch;
159
146
  def template_put(name, template)
160
147
  @client.indices.put_template(:name => name, :body => template)
161
148
  end
149
+
150
+ # Build a bulk item for an elasticsearch update action
151
+ def update_action_builder(args, source)
152
+ if args[:_script]
153
+ # Use the event as a hash from your script with variable name defined
154
+ # by script_var_name (default: "event")
155
+ # Ex: event["@timestamp"]
156
+ source = { 'script' => {'params' => { @options[:script_var_name] => source }} }
157
+ if @options[:scripted_upsert]
158
+ source['scripted_upsert'] = true
159
+ source['upsert'] = {}
160
+ else
161
+ source['upsert'] = args.delete(:_upsert) if args[:_upsert]
162
+ end
163
+ case @options[:script_type]
164
+ when 'indexed'
165
+ source['script']['id'] = args.delete(:_script)
166
+ when 'file'
167
+ source['script']['file'] = args.delete(:_script)
168
+ when 'inline'
169
+ source['script']['inline'] = args.delete(:_script)
170
+ end
171
+ source['script']['lang'] = @options[:script_lang] if @options[:script_lang] != ''
172
+ else
173
+ source = { 'doc' => source }
174
+ if @options[:doc_as_upsert]
175
+ source['doc_as_upsert'] = true
176
+ else
177
+ source['upsert'] = args.delete(:_upsert) if args[:_upsert]
178
+ end
179
+ end
180
+ [args, source]
181
+ end
162
182
  end
163
- end end end
183
+ end end end
@@ -17,10 +17,23 @@ module LogStash; module Outputs; class ElasticSearch;
17
17
  client_settings.merge! setup_proxy(logger, params)
18
18
  common_options.merge! setup_basic_auth(logger, params)
19
19
 
20
+ # Update API setup
21
+ raise( Logstash::ConfigurationError,
22
+ "doc_as_upsert and scripted_upsert are mutually exclusive."
23
+ ) if params["doc_as_upsert"] and params["scripted_upsert"]
24
+
25
+ raise(
26
+ LogStash::ConfigurationError,
27
+ "Specifying action => 'update' needs a document_id."
28
+ ) if params['action'] == 'update' and params.fetch('document_id', '') == ''
29
+
20
30
  # Update API setup
21
31
  update_options = {
22
- :upsert => params["upsert"],
23
- :doc_as_upsert => params["doc_as_upsert"]
32
+ :doc_as_upsert => params["doc_as_upsert"],
33
+ :script_var_name => params["script_var_name"],
34
+ :script_type => params["script_type"],
35
+ :script_lang => params["script_lang"],
36
+ :scripted_upsert => params["scripted_upsert"]
24
37
  }
25
38
  common_options.merge! update_options if params["action"] == 'update'
26
39
 
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-elasticsearch'
4
- s.version = '2.3.2'
4
+ s.version = '2.4.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Elasticsearch"
7
7
  s.description = "Output events to elasticsearch"
@@ -8,7 +8,7 @@ require "logstash/outputs/elasticsearch"
8
8
 
9
9
  CONTAINER_NAME = "logstash-output-elasticsearch-#{rand(999).to_s}"
10
10
  CONTAINER_IMAGE = "elasticsearch"
11
- CONTAINER_TAG = "1.6"
11
+ CONTAINER_TAG = "2.0"
12
12
 
13
13
  DOCKER_INTEGRATION = ENV["DOCKER_INTEGRATION"]
14
14
 
@@ -45,7 +45,14 @@ RSpec.configure do |config|
45
45
  ls = Longshoreman::new
46
46
  ls.container.get(CONTAINER_NAME)
47
47
  rescue Docker::Error::NotFoundError
48
- Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME)
48
+ scriptDir = File.expand_path File.dirname(__FILE__) + '/fixtures/scripts'
49
+ Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME, {
50
+ 'Cmd' => [ "-Des.script.inline=on", "-Des.script.indexed=on" ],
51
+ 'HostConfig' => {
52
+ 'Binds' => ["#{scriptDir}:/usr/share/elasticsearch/config/scripts"],
53
+ 'PublishAllPorts' => true
54
+ }
55
+ })
49
56
  # TODO(talevy): verify ES is running instead of static timeout
50
57
  sleep 10
51
58
  end
@@ -0,0 +1,2 @@
1
+ ctx._source.counter += event["count"]
2
+
@@ -0,0 +1,2 @@
1
+ ctx._source.counter += event["data"]["count"]
2
+
@@ -0,0 +1,2 @@
1
+ ctx._source.counter = event["counter"]
2
+
@@ -1,10 +1,10 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
 
3
- describe "all protocols update actions", :integration => true do
3
+ describe "Update actions", :integration => true do
4
4
  require "logstash/outputs/elasticsearch"
5
5
  require "elasticsearch"
6
6
 
7
- def get_es_output(id = nil, upsert = nil, doc_as_upsert=nil)
7
+ def get_es_output( options={} )
8
8
  settings = {
9
9
  "manage_template" => true,
10
10
  "index" => "logstash-update",
@@ -12,10 +12,7 @@ describe "all protocols update actions", :integration => true do
12
12
  "hosts" => get_host_port(),
13
13
  "action" => "update"
14
14
  }
15
- settings['upsert'] = upsert unless upsert.nil?
16
- settings['document_id'] = id unless id.nil?
17
- settings['doc_as_upsert'] = doc_as_upsert unless doc_as_upsert.nil?
18
- LogStash::Outputs::ElasticSearch.new(settings)
15
+ LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
19
16
  end
20
17
 
21
18
  before :each do
@@ -29,39 +26,87 @@ describe "all protocols update actions", :integration => true do
29
26
  :index => 'logstash-update',
30
27
  :type => 'logs',
31
28
  :id => "123",
32
- :body => { :message => 'Test' }
29
+ :body => { :message => 'Test', :counter => 1 }
33
30
  )
34
31
  @es.indices.refresh
35
32
  end
36
33
 
37
34
  it "should fail without a document_id" do
38
- event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
39
- action = ["update", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
40
35
  subject = get_es_output
41
- subject.register
42
- expect { subject.flush([action]) }.to raise_error
36
+ expect { subject.register }.to raise_error(LogStash::ConfigurationError)
43
37
  end
44
38
 
45
- it "should not create new document" do
46
- subject = get_es_output("456")
47
- subject.register
48
- subject.receive(LogStash::Event.new("message" => "sample message here"))
49
- subject.flush
50
- expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
39
+ context "when update only" do
40
+ it "should not create new document" do
41
+ subject = get_es_output({ 'document_id' => "456" } )
42
+ subject.register
43
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
44
+ subject.flush
45
+ expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
46
+ end
47
+
48
+ it "should update existing document" do
49
+ subject = get_es_output({ 'document_id' => "123" })
50
+ subject.register
51
+ subject.receive(LogStash::Event.new("message" => "updated message here"))
52
+ subject.flush
53
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
54
+ insist { r["_source"]["message"] } == 'updated message here'
55
+ end
51
56
  end
57
+
58
+ context "when using script" do
59
+ it "should increment a counter with event/doc 'count' variable" do
60
+ subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
61
+ subject.register
62
+ subject.receive(LogStash::Event.new("count" => 2))
63
+ subject.flush
64
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
65
+ insist { r["_source"]["counter"] } == 3
66
+ end
52
67
 
53
- it "should update existing document" do
54
- subject = get_es_output("123")
55
- subject.register
56
- subject.receive(LogStash::Event.new("message" => "updated message here"))
57
- subject.flush
58
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
59
- insist { r["_source"]["message"] } == 'updated message here'
68
+ it "should increment a counter with event/doc '[data][count]' nested variable" do
69
+ subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
70
+ subject.register
71
+ subject.receive(LogStash::Event.new("data" => { "count" => 3 }))
72
+ subject.flush
73
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
74
+ insist { r["_source"]["counter"] } == 4
75
+ end
76
+
77
+ it "should increment a counter with event/doc 'count' variable with inline script" do
78
+ subject = get_es_output({
79
+ 'document_id' => "123",
80
+ 'script' => 'ctx._source.counter += event["count"]',
81
+ 'script_lang' => 'groovy',
82
+ 'script_type' => 'inline'
83
+ })
84
+ subject.register
85
+ subject.receive(LogStash::Event.new("count" => 3 ))
86
+ subject.flush
87
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
88
+ insist { r["_source"]["counter"] } == 4
89
+ end
90
+
91
+ it "should increment a counter with event/doc 'count' variable with indexed script" do
92
+ @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
93
+ subject = get_es_output({
94
+ 'document_id' => "123",
95
+ 'script' => 'indexed_update',
96
+ 'script_lang' => 'groovy',
97
+ 'script_type' => 'indexed'
98
+ })
99
+ subject.register
100
+ subject.receive(LogStash::Event.new("count" => 4 ))
101
+ subject.flush
102
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
103
+ insist { r["_source"]["counter"] } == 5
104
+ end
60
105
  end
61
106
 
62
- context "upsert with protocol" do
63
- it "should create new documents with upsert content" do
64
- subject = get_es_output("456", '{"message": "upsert message"}')
107
+ context "when update with upsert" do
108
+ it "should create new documents with provided upsert" do
109
+ subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
65
110
  subject.register
66
111
  subject.receive(LogStash::Event.new("message" => "sample message here"))
67
112
  subject.flush
@@ -70,12 +115,32 @@ describe "all protocols update actions", :integration => true do
70
115
  end
71
116
 
72
117
  it "should create new documents with event/doc as upsert" do
73
- subject = get_es_output("456", nil, true)
118
+ subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
74
119
  subject.register
75
120
  subject.receive(LogStash::Event.new("message" => "sample message here"))
76
121
  subject.flush
77
122
  r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
78
123
  insist { r["_source"]["message"] } == 'sample message here'
79
124
  end
125
+
126
+ context "when using script" do
127
+ it "should create new documents with upsert content" do
128
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
129
+ subject.register
130
+ subject.receive(LogStash::Event.new("message" => "sample message here"))
131
+ subject.flush
132
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
133
+ insist { r["_source"]["message"] } == 'upsert message'
134
+ end
135
+
136
+ it "should create new documents with event/doc as script params" do
137
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
138
+ subject.register
139
+ subject.receive(LogStash::Event.new("counter" => 1))
140
+ subject.flush
141
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
142
+ insist { r["_source"]["counter"] } == 1
143
+ end
144
+ end
80
145
  end
81
146
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.2
4
+ version: 2.4.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-01-05 00:00:00.000000000 Z
11
+ date: 2016-01-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
@@ -210,6 +210,9 @@ files:
210
210
  - lib/logstash/outputs/elasticsearch/template_manager.rb
211
211
  - logstash-output-elasticsearch.gemspec
212
212
  - spec/es_spec_helper.rb
213
+ - spec/fixtures/scripts/scripted_update.groovy
214
+ - spec/fixtures/scripts/scripted_update_nested.groovy
215
+ - spec/fixtures/scripts/scripted_upsert.groovy
213
216
  - spec/integration/outputs/create_spec.rb
214
217
  - spec/integration/outputs/index_spec.rb
215
218
  - spec/integration/outputs/parent_spec.rb
@@ -252,6 +255,9 @@ specification_version: 4
252
255
  summary: Logstash Output to Elasticsearch
253
256
  test_files:
254
257
  - spec/es_spec_helper.rb
258
+ - spec/fixtures/scripts/scripted_update.groovy
259
+ - spec/fixtures/scripts/scripted_update_nested.groovy
260
+ - spec/fixtures/scripts/scripted_upsert.groovy
255
261
  - spec/integration/outputs/create_spec.rb
256
262
  - spec/integration/outputs/index_spec.rb
257
263
  - spec/integration/outputs/parent_spec.rb