logstash-output-elasticsearch 10.8.6-java → 11.0.3-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +17 -0
  3. data/docs/index.asciidoc +132 -22
  4. data/lib/logstash/outputs/elasticsearch.rb +125 -64
  5. data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +233 -0
  6. data/lib/logstash/outputs/elasticsearch/http_client.rb +9 -7
  7. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +49 -62
  8. data/lib/logstash/outputs/elasticsearch/ilm.rb +13 -45
  9. data/lib/logstash/outputs/elasticsearch/license_checker.rb +26 -23
  10. data/lib/logstash/outputs/elasticsearch/template_manager.rb +4 -6
  11. data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json +1 -0
  12. data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +157 -153
  13. data/lib/logstash/plugin_mixins/elasticsearch/common.rb +71 -58
  14. data/logstash-output-elasticsearch.gemspec +3 -3
  15. data/spec/es_spec_helper.rb +7 -12
  16. data/spec/fixtures/_nodes/{5x_6x.json → 6x.json} +5 -5
  17. data/spec/integration/outputs/compressed_indexing_spec.rb +47 -46
  18. data/spec/integration/outputs/data_stream_spec.rb +61 -0
  19. data/spec/integration/outputs/delete_spec.rb +49 -51
  20. data/spec/integration/outputs/ilm_spec.rb +236 -248
  21. data/spec/integration/outputs/index_spec.rb +5 -2
  22. data/spec/integration/outputs/index_version_spec.rb +78 -82
  23. data/spec/integration/outputs/ingest_pipeline_spec.rb +58 -58
  24. data/spec/integration/outputs/painless_update_spec.rb +74 -164
  25. data/spec/integration/outputs/parent_spec.rb +67 -75
  26. data/spec/integration/outputs/retry_spec.rb +6 -6
  27. data/spec/integration/outputs/sniffer_spec.rb +15 -54
  28. data/spec/integration/outputs/templates_spec.rb +79 -81
  29. data/spec/integration/outputs/update_spec.rb +99 -101
  30. data/spec/spec_helper.rb +10 -0
  31. data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +528 -0
  32. data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +1 -0
  33. data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +36 -29
  34. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +2 -3
  35. data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +10 -12
  36. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +1 -2
  37. data/spec/unit/outputs/elasticsearch_spec.rb +176 -41
  38. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +1 -2
  39. data/spec/unit/outputs/error_whitelist_spec.rb +3 -2
  40. data/spec/unit/outputs/license_check_spec.rb +0 -16
  41. metadata +29 -36
  42. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json +0 -95
  43. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json +0 -46
  44. data/spec/fixtures/_nodes/2x_1x.json +0 -27
  45. data/spec/fixtures/scripts/groovy/scripted_update.groovy +0 -2
  46. data/spec/fixtures/scripts/groovy/scripted_update_nested.groovy +0 -2
  47. data/spec/fixtures/scripts/groovy/scripted_upsert.groovy +0 -2
  48. data/spec/integration/outputs/groovy_update_spec.rb +0 -150
  49. data/spec/integration/outputs/templates_5x_spec.rb +0 -98
@@ -80,8 +80,11 @@ describe "indexing" do
80
80
  response = http_client.get("#{index_url}/_search?q=*&size=1000")
81
81
  result = LogStash::Json.load(response.body)
82
82
  result["hits"]["hits"].each do |doc|
83
- expect(doc["_type"]).to eq(type) if ESHelper.es_version_satisfies?(">= 6", "< 8")
84
- expect(doc).not_to include("_type") if ESHelper.es_version_satisfies?(">= 8")
83
+ if ESHelper.es_version_satisfies?("< 8")
84
+ expect(doc["_type"]).to eq(type)
85
+ else
86
+ expect(doc).not_to include("_type")
87
+ end
85
88
  expect(doc["_index"]).to eq(index)
86
89
  end
87
90
  end
@@ -1,101 +1,97 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
  require "logstash/outputs/elasticsearch"
3
3
 
4
- if ESHelper.es_version_satisfies?(">= 2")
5
- describe "Versioned indexing", :integration => true do
6
- require "logstash/outputs/elasticsearch"
4
+ describe "Versioned indexing", :integration => true do
5
+ require "logstash/outputs/elasticsearch"
7
6
 
8
- let(:es) { get_client }
7
+ let(:es) { get_client }
9
8
 
10
- before :each do
11
- # Delete all templates first.
12
- # Clean ES of data before we start.
13
- es.indices.delete_template(:name => "*")
14
- # This can fail if there are no indexes, ignore failure.
15
- es.indices.delete(:index => "*") rescue nil
16
- es.indices.refresh
17
- end
9
+ before :each do
10
+ # Delete all templates first.
11
+ # Clean ES of data before we start.
12
+ es.indices.delete_template(:name => "*")
13
+ # This can fail if there are no indexes, ignore failure.
14
+ es.indices.delete(:index => "*") rescue nil
15
+ es.indices.refresh
16
+ end
18
17
 
19
- context "when index only" do
20
- subject { LogStash::Outputs::ElasticSearch.new(settings) }
18
+ context "when index only" do
19
+ subject { LogStash::Outputs::ElasticSearch.new(settings) }
20
+
21
+ before do
22
+ subject.register
23
+ end
21
24
 
22
- before do
23
- subject.register
25
+ describe "unversioned output" do
26
+ let(:settings) do
27
+ {
28
+ "manage_template" => true,
29
+ "index" => "logstash-index",
30
+ "template_overwrite" => true,
31
+ "hosts" => get_host_port(),
32
+ "action" => "index",
33
+ "document_id" => "%{my_id}"
34
+ }
24
35
  end
25
36
 
26
- describe "unversioned output" do
27
- let(:settings) do
28
- {
29
- "manage_template" => true,
30
- "index" => "logstash-index",
31
- "template_overwrite" => true,
32
- "hosts" => get_host_port(),
33
- "action" => "index",
34
- "script_lang" => "groovy",
35
- "document_id" => "%{my_id}"
36
- }
37
- end
37
+ it "should default to ES version" do
38
+ subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
39
+ r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
40
+ expect(r["_version"]).to eq(1)
41
+ expect(r["_source"]["message"]).to eq('foo')
42
+ subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
43
+ r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
44
+ expect(r2["_version"]).to eq(2)
45
+ expect(r2["_source"]["message"]).to eq('foobar')
46
+ end
47
+ end
38
48
 
39
- it "should default to ES version" do
40
- subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
41
- r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
42
- expect(r["_version"]).to eq(1)
43
- expect(r["_source"]["message"]).to eq('foo')
44
- subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
45
- r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
46
- expect(r2["_version"]).to eq(2)
47
- expect(r2["_source"]["message"]).to eq('foobar')
48
- end
49
+ describe "versioned output" do
50
+ let(:settings) do
51
+ {
52
+ "manage_template" => true,
53
+ "index" => "logstash-index",
54
+ "template_overwrite" => true,
55
+ "hosts" => get_host_port(),
56
+ "action" => "index",
57
+ "document_id" => "%{my_id}",
58
+ "version" => "%{my_version}",
59
+ "version_type" => "external",
60
+ }
49
61
  end
50
62
 
51
- describe "versioned output" do
52
- let(:settings) do
53
- {
54
- "manage_template" => true,
55
- "index" => "logstash-index",
56
- "template_overwrite" => true,
57
- "hosts" => get_host_port(),
58
- "action" => "index",
59
- "script_lang" => "groovy",
60
- "document_id" => "%{my_id}",
61
- "version" => "%{my_version}",
62
- "version_type" => "external",
63
- }
64
- end
65
-
66
- it "should respect the external version" do
67
- id = "ev1"
68
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
69
- r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
70
- expect(r["_version"]).to eq(99)
71
- expect(r["_source"]["message"]).to eq('foo')
72
- end
63
+ it "should respect the external version" do
64
+ id = "ev1"
65
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
66
+ r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
67
+ expect(r["_version"]).to eq(99)
68
+ expect(r["_source"]["message"]).to eq('foo')
69
+ end
73
70
 
74
- it "should ignore non-monotonic external version updates" do
75
- id = "ev2"
76
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
77
- r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
78
- expect(r["_version"]).to eq(99)
79
- expect(r["_source"]["message"]).to eq('foo')
71
+ it "should ignore non-monotonic external version updates" do
72
+ id = "ev2"
73
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
74
+ r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
75
+ expect(r["_version"]).to eq(99)
76
+ expect(r["_source"]["message"]).to eq('foo')
80
77
 
81
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
82
- r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
83
- expect(r2["_version"]).to eq(99)
84
- expect(r2["_source"]["message"]).to eq('foo')
85
- end
78
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
79
+ r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
80
+ expect(r2["_version"]).to eq(99)
81
+ expect(r2["_source"]["message"]).to eq('foo')
82
+ end
86
83
 
87
- it "should commit monotonic external version updates" do
88
- id = "ev3"
89
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
90
- r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
91
- expect(r["_version"]).to eq(99)
92
- expect(r["_source"]["message"]).to eq('foo')
84
+ it "should commit monotonic external version updates" do
85
+ id = "ev3"
86
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
87
+ r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
88
+ expect(r["_version"]).to eq(99)
89
+ expect(r["_source"]["message"]).to eq('foo')
93
90
 
94
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
95
- r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
96
- expect(r2["_version"]).to eq(100)
97
- expect(r2["_source"]["message"]).to eq('foo')
98
- end
91
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
92
+ r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
93
+ expect(r2["_version"]).to eq(100)
94
+ expect(r2["_source"]["message"]).to eq('foo')
99
95
  end
100
96
  end
101
97
  end
@@ -1,74 +1,74 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
 
3
- if ESHelper.es_version_satisfies?(">= 5")
4
- describe "Ingest pipeline execution behavior", :integration => true do
5
- subject! do
6
- require "logstash/outputs/elasticsearch"
7
- settings = {
8
- "hosts" => "#{get_host_port()}",
9
- "pipeline" => "apache-logs"
10
- }
11
- next LogStash::Outputs::ElasticSearch.new(settings)
12
- end
3
+ describe "Ingest pipeline execution behavior", :integration => true do
4
+ subject! do
5
+ require "logstash/outputs/elasticsearch"
6
+ settings = {
7
+ "hosts" => "#{get_host_port()}",
8
+ "pipeline" => "apache-logs",
9
+ "data_stream" => 'false'
10
+ }
11
+ next LogStash::Outputs::ElasticSearch.new(settings)
12
+ end
13
13
 
14
- let(:http_client) { Manticore::Client.new }
15
- let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
16
- let(:apache_logs_pipeline) { '
17
- {
18
- "description" : "Pipeline to parse Apache logs",
19
- "processors" : [
20
- {
21
- "grok": {
22
- "field": "message",
23
- "patterns": ["%{COMBINEDAPACHELOG}"]
24
- }
14
+ let(:http_client) { Manticore::Client.new }
15
+ let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
16
+ let(:apache_logs_pipeline) { '
17
+ {
18
+ "description" : "Pipeline to parse Apache logs",
19
+ "processors" : [
20
+ {
21
+ "grok": {
22
+ "field": "message",
23
+ "patterns": ["%{COMBINEDAPACHELOG}"]
25
24
  }
26
- ]
27
- }'
28
- }
25
+ }
26
+ ]
27
+ }'
28
+ }
29
29
 
30
- before :each do
31
- # Delete all templates first.
32
- require "elasticsearch"
30
+ before :each do
31
+ # Delete all templates first.
32
+ require "elasticsearch"
33
33
 
34
- # Clean ES of data before we start.
35
- @es = get_client
36
- @es.indices.delete_template(:name => "*")
34
+ # Clean ES of data before we start.
35
+ @es = get_client
36
+ @es.indices.delete_template(:name => "*")
37
37
 
38
- # This can fail if there are no indexes, ignore failure.
39
- @es.indices.delete(:index => "*") rescue nil
38
+ # This can fail if there are no indexes, ignore failure.
39
+ @es.indices.delete(:index => "*") rescue nil
40
40
 
41
- # delete existing ingest pipeline
42
- http_client.delete(ingest_url).call
41
+ # delete existing ingest pipeline
42
+ http_client.delete(ingest_url).call
43
43
 
44
- # register pipeline
45
- http_client.put(ingest_url, :body => apache_logs_pipeline, :headers => {"Content-Type" => "application/json" }).call
44
+ # register pipeline
45
+ http_client.put(ingest_url, :body => apache_logs_pipeline, :headers => {"Content-Type" => "application/json" }).call
46
46
 
47
- #TODO: Use esclient
48
- #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
47
+ #TODO: Use esclient
48
+ #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
49
49
 
50
- subject.register
51
- subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
52
- @es.indices.refresh
50
+ subject.register
51
+ subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
52
+ @es.indices.refresh
53
53
 
54
- #Wait or fail until everything's indexed.
55
- Stud::try(20.times) do
56
- r = @es.search(index: 'logstash-*')
57
- expect(r).to have_hits(1)
58
- end
54
+ #Wait or fail until everything's indexed.
55
+ Stud::try(10.times) do
56
+ r = @es.search(index: 'logstash-*')
57
+ expect(r).to have_hits(1)
58
+ sleep(0.1)
59
59
  end
60
+ end
60
61
 
61
- it "indexes using the proper pipeline" do
62
- results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
63
- expect(results).to have_hits(1)
64
- expect(results["hits"]["hits"][0]["_source"]["response"]).to eq("200")
65
- expect(results["hits"]["hits"][0]["_source"]["bytes"]).to eq("182")
66
- expect(results["hits"]["hits"][0]["_source"]["verb"]).to eq("GET")
67
- expect(results["hits"]["hits"][0]["_source"]["request"]).to eq("/scripts/netcat-webserver")
68
- expect(results["hits"]["hits"][0]["_source"]["auth"]).to eq("-")
69
- expect(results["hits"]["hits"][0]["_source"]["ident"]).to eq("-")
70
- expect(results["hits"]["hits"][0]["_source"]["clientip"]).to eq("183.60.215.50")
71
- expect(results["hits"]["hits"][0]["_source"]["junkfieldaaaa"]).to eq(nil)
72
- end
62
+ it "indexes using the proper pipeline" do
63
+ results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
64
+ expect(results).to have_hits(1)
65
+ expect(results["hits"]["hits"][0]["_source"]["response"]).to eq("200")
66
+ expect(results["hits"]["hits"][0]["_source"]["bytes"]).to eq("182")
67
+ expect(results["hits"]["hits"][0]["_source"]["verb"]).to eq("GET")
68
+ expect(results["hits"]["hits"][0]["_source"]["request"]).to eq("/scripts/netcat-webserver")
69
+ expect(results["hits"]["hits"][0]["_source"]["auth"]).to eq("-")
70
+ expect(results["hits"]["hits"][0]["_source"]["ident"]).to eq("-")
71
+ expect(results["hits"]["hits"][0]["_source"]["clientip"]).to eq("183.60.215.50")
72
+ expect(results["hits"]["hits"][0]["_source"]["junkfieldaaaa"]).to eq(nil)
73
73
  end
74
74
  end
@@ -1,188 +1,98 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
 
3
- if ESHelper.es_version_satisfies?(">= 5")
4
- describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
5
- require "logstash/outputs/elasticsearch"
6
-
7
- def get_es_output( options={} )
8
- settings = {
9
- "manage_template" => true,
10
- "index" => "logstash-update",
11
- "template_overwrite" => true,
12
- "hosts" => get_host_port(),
13
- "action" => "update"
14
- }
15
- if ESHelper.es_version_satisfies?('<6')
16
- settings.merge!({"script_lang" => "painless"})
17
- end
18
- LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
19
- end
20
-
21
- before :each do
22
- @es = get_client
23
- # Delete all templates first.
24
- # Clean ES of data before we start.
25
- @es.indices.delete_template(:name => "*")
26
- # This can fail if there are no indexes, ignore failure.
27
- @es.indices.delete(:index => "*") rescue nil
28
- @es.index(
29
- :index => 'logstash-update',
30
- :type => doc_type,
31
- :id => "123",
32
- :body => { :message => 'Test', :counter => 1 }
33
- )
34
- @es.indices.refresh
35
- end
3
+ describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
4
+ require "logstash/outputs/elasticsearch"
5
+
6
+ def get_es_output( options={} )
7
+ settings = {
8
+ "manage_template" => true,
9
+ "index" => "logstash-update",
10
+ "template_overwrite" => true,
11
+ "hosts" => get_host_port(),
12
+ "action" => "update"
13
+ }
14
+ LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
15
+ end
36
16
 
37
- context "scripted updates" do
38
- if ESHelper.es_version_satisfies?('<6')
39
- context 'with file based scripts' do
40
- it "should increment a counter with event/doc 'count' variable" do
41
- subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
42
- subject.register
43
- subject.multi_receive([LogStash::Event.new("count" => 2)])
44
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
45
- expect(r["_source"]["counter"]).to eq(3)
46
- end
17
+ before :each do
18
+ @es = get_client
19
+ # Delete all templates first.
20
+ # Clean ES of data before we start.
21
+ @es.indices.delete_template(:name => "*")
22
+ # This can fail if there are no indexes, ignore failure.
23
+ @es.indices.delete(:index => "*") rescue nil
24
+ @es.index(
25
+ :index => 'logstash-update',
26
+ :type => doc_type,
27
+ :id => "123",
28
+ :body => { :message => 'Test', :counter => 1 }
29
+ )
30
+ @es.indices.refresh
31
+ end
47
32
 
48
- it "should increment a counter with event/doc '[data][count]' nested variable" do
49
- subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
50
- subject.register
51
- subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
52
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
53
- expect(r["_source"]["counter"]).to eq(4)
54
- end
55
- end
56
- end
33
+ context "scripted updates" do
34
+ context 'with an indexed script' do
35
+ it "should increment a counter with event/doc 'count' variable with indexed script" do
36
+ @es.perform_request(:put, "_scripts/indexed_update", {}, {"script" => {"source" => "ctx._source.counter += params.event.count", "lang" => "painless"}})
57
37
 
58
- it "should increment a counter with event/doc 'count' variable with inline script" do
59
- subject = get_es_output({
38
+ plugin_parameters = {
60
39
  'document_id' => "123",
61
- 'script' => 'ctx._source.counter += params.event.counter',
62
- 'script_type' => 'inline'
63
- })
64
- subject.register
65
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
66
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
67
- expect(r["_source"]["counter"]).to eq(4)
68
- end
40
+ 'script' => 'indexed_update',
41
+ 'script_type' => 'indexed'
42
+ }
69
43
 
70
- it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
71
- subject = get_es_output({
72
- 'document_id' => "123",
73
- 'doc_as_upsert' => true,
74
- 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
75
- 'script_type' => 'inline'
76
- })
77
- subject.register
78
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
79
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
80
- expect(r["_source"]["counter"]).to eq(4)
81
- end
44
+ plugin_parameters.merge!('script_lang' => '')
82
45
 
83
- it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
84
- subject = get_es_output({
85
- 'document_id' => "456",
86
- 'doc_as_upsert' => true,
87
- 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
88
- 'script_type' => 'inline'
89
- })
46
+ subject = get_es_output(plugin_parameters)
90
47
  subject.register
91
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
92
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
93
- expect(r["_source"]["counter"]).to eq(3)
48
+ subject.multi_receive([LogStash::Event.new("count" => 4 )])
49
+ r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
50
+ expect(r["_source"]["counter"]).to eq(5)
94
51
  end
52
+ end
53
+ end
54
+
55
+ context "when update with upsert" do
56
+ it "should create new documents with provided upsert" do
57
+ subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
58
+ subject.register
59
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
60
+ r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
61
+ expect(r["_source"]["message"]).to eq('upsert message')
62
+ end
95
63
 
96
- context 'with an indexed script' do
97
- it "should increment a counter with event/doc 'count' variable with indexed script" do
98
- if ESHelper.es_version_satisfies?('<6')
99
- @es.perform_request(:put, "_scripts/painless/indexed_update", {}, {"script" => "ctx._source.counter += params.event.count" })
100
- else
101
- @es.perform_request(:put, "_scripts/indexed_update", {}, {"script" => {"source" => "ctx._source.counter += params.event.count", "lang" => "painless"}})
102
- end
103
-
104
- plugin_parameters = {
105
- 'document_id' => "123",
106
- 'script' => 'indexed_update',
107
- 'script_type' => 'indexed'
108
- }
109
-
110
- if ESHelper.es_version_satisfies?('>= 6.0.0')
111
- plugin_parameters.merge!('script_lang' => '')
112
- end
64
+ it "should create new documents with event/doc as upsert" do
65
+ subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
66
+ subject.register
67
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
68
+ r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
69
+ expect(r["_source"]["message"]).to eq('sample message here')
70
+ end
113
71
 
114
- subject = get_es_output(plugin_parameters)
115
- subject.register
116
- subject.multi_receive([LogStash::Event.new("count" => 4 )])
117
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
118
- expect(r["_source"]["counter"]).to eq(5)
119
- end
120
- end
121
- end
72
+ it "should fail on documents with event/doc as upsert at external version" do
73
+ subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
74
+ expect { subject.register }.to raise_error(LogStash::ConfigurationError)
75
+ end
76
+ end
122
77
 
123
- context "when update with upsert" do
124
- it "should create new documents with provided upsert" do
125
- subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
78
+ context "updates with scripted upsert" do
79
+ context 'with an inline script' do
80
+ it "should create new documents with upsert content" do
81
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
126
82
  subject.register
83
+
127
84
  subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
128
85
  r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
129
86
  expect(r["_source"]["message"]).to eq('upsert message')
130
87
  end
131
88
 
132
- it "should create new documents with event/doc as upsert" do
133
- subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
89
+ it "should create new documents with event/doc as script params" do
90
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
134
91
  subject.register
135
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
92
+ subject.multi_receive([LogStash::Event.new("counter" => 1)])
93
+ @es.indices.refresh
136
94
  r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
137
- expect(r["_source"]["message"]).to eq('sample message here')
138
- end
139
-
140
- it "should fail on documents with event/doc as upsert at external version" do
141
- subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
142
- expect { subject.register }.to raise_error(LogStash::ConfigurationError)
143
- end
144
- end
145
-
146
- context "updates with scripted upsert" do
147
- if ESHelper.es_version_satisfies?('<6')
148
- context 'with file based scripts' do
149
- it "should create new documents with upsert content" do
150
- subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
151
- subject.register
152
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
153
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
154
- expect(r["_source"]["message"]).to eq('upsert message')
155
- end
156
-
157
- it "should create new documents with event/doc as script params" do
158
- subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
159
- subject.register
160
- subject.multi_receive([LogStash::Event.new("counter" => 1)])
161
- @es.indices.refresh
162
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
163
- expect(r["_source"]["counter"]).to eq(1)
164
- end
165
- end
166
- end
167
-
168
- context 'with an inline script' do
169
- it "should create new documents with upsert content" do
170
- subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
171
- subject.register
172
-
173
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
174
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
175
- expect(r["_source"]["message"]).to eq('upsert message')
176
- end
177
-
178
- it "should create new documents with event/doc as script params" do
179
- subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
180
- subject.register
181
- subject.multi_receive([LogStash::Event.new("counter" => 1)])
182
- @es.indices.refresh
183
- r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
184
- expect(r["_source"]["counter"]).to eq(1)
185
- end
95
+ expect(r["_source"]["counter"]).to eq(1)
186
96
  end
187
97
  end
188
98
  end