logstash-output-elasticsearch 7.4.0-java → 7.4.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,148 +1,150 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
 
3
- describe "Update actions using groovy scripts", :integration => true, :update_tests => 'groovy', :version_greater_than_equal_to_2x => true do
4
- require "logstash/outputs/elasticsearch"
3
+ if ESHelper.es_version_satisfies?(">= 2")
4
+ describe "Update actions using groovy scripts", :integration => true, :update_tests => 'groovy' do
5
+ require "logstash/outputs/elasticsearch"
5
6
 
6
- def get_es_output( options={} )
7
- settings = {
8
- "manage_template" => true,
9
- "index" => "logstash-update",
10
- "template_overwrite" => true,
11
- "hosts" => get_host_port(),
12
- "action" => "update",
13
- "script_lang" => "groovy"
14
- }
15
- LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
16
- end
17
-
18
- before :each do
19
- @es = get_client
20
- # Delete all templates first.
21
- # Clean ES of data before we start.
22
- @es.indices.delete_template(:name => "*")
23
- # This can fail if there are no indexes, ignore failure.
24
- @es.indices.delete(:index => "*") rescue nil
25
- @es.index(
26
- :index => 'logstash-update',
27
- :type => 'logs',
28
- :id => "123",
29
- :body => { :message => 'Test', :counter => 1 }
30
- )
31
- @es.indices.refresh
32
- end
33
-
34
- context "scripted updates" do
35
- it "should increment a counter with event/doc 'count' variable" do
36
- subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
37
- subject.register
38
- subject.multi_receive([LogStash::Event.new("count" => 2)])
39
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
40
- insist { r["_source"]["counter"] } == 3
7
+ def get_es_output( options={} )
8
+ settings = {
9
+ "manage_template" => true,
10
+ "index" => "logstash-update",
11
+ "template_overwrite" => true,
12
+ "hosts" => get_host_port(),
13
+ "action" => "update",
14
+ "script_lang" => "groovy"
15
+ }
16
+ LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
41
17
  end
42
18
 
43
- it "should increment a counter with event/doc '[data][count]' nested variable" do
44
- subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
45
- subject.register
46
- subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
47
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
48
- insist { r["_source"]["counter"] } == 4
19
+ before :each do
20
+ @es = get_client
21
+ # Delete all templates first.
22
+ # Clean ES of data before we start.
23
+ @es.indices.delete_template(:name => "*")
24
+ # This can fail if there are no indexes, ignore failure.
25
+ @es.indices.delete(:index => "*") rescue nil
26
+ @es.index(
27
+ :index => 'logstash-update',
28
+ :type => 'logs',
29
+ :id => "123",
30
+ :body => { :message => 'Test', :counter => 1 }
31
+ )
32
+ @es.indices.refresh
49
33
  end
50
34
 
51
- it "should increment a counter with event/doc 'count' variable with inline script" do
52
- subject = get_es_output({
53
- 'document_id' => "123",
54
- 'script' => 'ctx._source.counter += event["counter"]',
55
- 'script_lang' => 'groovy',
56
- 'script_type' => 'inline'
57
- })
58
- subject.register
59
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
60
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
61
- insist { r["_source"]["counter"] } == 4
62
- end
35
+ context "scripted updates" do
36
+ it "should increment a counter with event/doc 'count' variable" do
37
+ subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
38
+ subject.register
39
+ subject.multi_receive([LogStash::Event.new("count" => 2)])
40
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
41
+ insist { r["_source"]["counter"] } == 3
42
+ end
63
43
 
64
- it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
65
- subject = get_es_output({
66
- 'document_id' => "123",
67
- 'doc_as_upsert' => true,
68
- 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
69
- 'script_lang' => 'groovy',
70
- 'script_type' => 'inline'
71
- })
72
- subject.register
73
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
74
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
75
- insist { r["_source"]["counter"] } == 4
76
- end
44
+ it "should increment a counter with event/doc '[data][count]' nested variable" do
45
+ subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
46
+ subject.register
47
+ subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
48
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
49
+ insist { r["_source"]["counter"] } == 4
50
+ end
77
51
 
78
- it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
79
- subject = get_es_output({
80
- 'document_id' => "456",
81
- 'doc_as_upsert' => true,
82
- 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
83
- 'script_lang' => 'groovy',
84
- 'script_type' => 'inline'
85
- })
86
- subject.register
87
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
88
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
89
- insist { r["_source"]["counter"] } == 3
90
- end
52
+ it "should increment a counter with event/doc 'count' variable with inline script" do
53
+ subject = get_es_output({
54
+ 'document_id' => "123",
55
+ 'script' => 'ctx._source.counter += event["counter"]',
56
+ 'script_lang' => 'groovy',
57
+ 'script_type' => 'inline'
58
+ })
59
+ subject.register
60
+ subject.multi_receive([LogStash::Event.new("counter" => 3 )])
61
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
62
+ insist { r["_source"]["counter"] } == 4
63
+ end
91
64
 
92
- it "should increment a counter with event/doc 'count' variable with indexed script" do
93
- @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
94
- subject = get_es_output({
95
- 'document_id' => "123",
96
- 'script' => 'indexed_update',
97
- 'script_lang' => 'groovy',
98
- 'script_type' => 'indexed'
99
- })
100
- subject.register
101
- subject.multi_receive([LogStash::Event.new("count" => 4 )])
102
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
103
- insist { r["_source"]["counter"] } == 5
104
- end
105
- end
106
-
107
- context "when update with upsert" do
108
- it "should create new documents with provided upsert" do
109
- subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
110
- subject.register
111
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
112
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
113
- insist { r["_source"]["message"] } == 'upsert message'
114
- end
65
+ it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
66
+ subject = get_es_output({
67
+ 'document_id' => "123",
68
+ 'doc_as_upsert' => true,
69
+ 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
70
+ 'script_lang' => 'groovy',
71
+ 'script_type' => 'inline'
72
+ })
73
+ subject.register
74
+ subject.multi_receive([LogStash::Event.new("counter" => 3 )])
75
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
76
+ insist { r["_source"]["counter"] } == 4
77
+ end
115
78
 
116
- it "should create new documents with event/doc as upsert" do
117
- subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
118
- subject.register
119
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
120
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
121
- insist { r["_source"]["message"] } == 'sample message here'
122
- end
79
+ it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
80
+ subject = get_es_output({
81
+ 'document_id' => "456",
82
+ 'doc_as_upsert' => true,
83
+ 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
84
+ 'script_lang' => 'groovy',
85
+ 'script_type' => 'inline'
86
+ })
87
+ subject.register
88
+ subject.multi_receive([LogStash::Event.new("counter" => 3 )])
89
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
90
+ insist { r["_source"]["counter"] } == 3
91
+ end
123
92
 
124
- it "should fail on documents with event/doc as upsert at external version" do
125
- subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
126
- expect { subject.register }.to raise_error(LogStash::ConfigurationError)
93
+ it "should increment a counter with event/doc 'count' variable with indexed script" do
94
+ @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
95
+ subject = get_es_output({
96
+ 'document_id' => "123",
97
+ 'script' => 'indexed_update',
98
+ 'script_lang' => 'groovy',
99
+ 'script_type' => 'indexed'
100
+ })
101
+ subject.register
102
+ subject.multi_receive([LogStash::Event.new("count" => 4 )])
103
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
104
+ insist { r["_source"]["counter"] } == 5
105
+ end
127
106
  end
128
- end
129
107
 
130
- context "updates with scripted upsert" do
131
- it "should create new documents with upsert content" do
132
- subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
133
- subject.register
134
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
135
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
136
- insist { r["_source"]["message"] } == 'upsert message'
108
+ context "when update with upsert" do
109
+ it "should create new documents with provided upsert" do
110
+ subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
111
+ subject.register
112
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
113
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
114
+ insist { r["_source"]["message"] } == 'upsert message'
115
+ end
116
+
117
+ it "should create new documents with event/doc as upsert" do
118
+ subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
119
+ subject.register
120
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
121
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
122
+ insist { r["_source"]["message"] } == 'sample message here'
123
+ end
124
+
125
+ it "should fail on documents with event/doc as upsert at external version" do
126
+ subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
127
+ expect { subject.register }.to raise_error(LogStash::ConfigurationError)
128
+ end
137
129
  end
138
130
 
139
- it "should create new documents with event/doc as script params" do
140
- subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
141
- subject.register
142
- subject.multi_receive([LogStash::Event.new("counter" => 1)])
143
- @es.indices.refresh
144
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
145
- insist { r["_source"]["counter"] } == 1
131
+ context "updates with scripted upsert" do
132
+ it "should create new documents with upsert content" do
133
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
134
+ subject.register
135
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
136
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
137
+ insist { r["_source"]["message"] } == 'upsert message'
138
+ end
139
+
140
+ it "should create new documents with event/doc as script params" do
141
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
142
+ subject.register
143
+ subject.multi_receive([LogStash::Event.new("counter" => 1)])
144
+ @es.indices.refresh
145
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
146
+ insist { r["_source"]["counter"] } == 1
147
+ end
146
148
  end
147
149
  end
148
150
  end
@@ -1,100 +1,101 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
  require "logstash/outputs/elasticsearch"
3
3
 
4
+ if ESHelper.es_version_satisfies?(">= 2")
5
+ describe "Versioned indexing", :integration => true do
6
+ require "logstash/outputs/elasticsearch"
4
7
 
5
- describe "Versioned indexing", :integration => true, :version_greater_than_equal_to_2x => true do
6
- require "logstash/outputs/elasticsearch"
8
+ let(:es) { get_client }
7
9
 
8
- let(:es) { get_client }
9
-
10
- before :each do
11
- # Delete all templates first.
12
- # Clean ES of data before we start.
13
- es.indices.delete_template(:name => "*")
14
- # This can fail if there are no indexes, ignore failure.
15
- es.indices.delete(:index => "*") rescue nil
16
- es.indices.refresh
17
- end
18
-
19
- context "when index only" do
20
- subject { LogStash::Outputs::ElasticSearch.new(settings) }
21
-
22
- before do
23
- subject.register
10
+ before :each do
11
+ # Delete all templates first.
12
+ # Clean ES of data before we start.
13
+ es.indices.delete_template(:name => "*")
14
+ # This can fail if there are no indexes, ignore failure.
15
+ es.indices.delete(:index => "*") rescue nil
16
+ es.indices.refresh
24
17
  end
25
18
 
26
- describe "unversioned output" do
27
- let(:settings) do
28
- {
29
- "manage_template" => true,
30
- "index" => "logstash-index",
31
- "template_overwrite" => true,
32
- "hosts" => get_host_port(),
33
- "action" => "index",
34
- "script_lang" => "groovy",
35
- "document_id" => "%{my_id}"
36
- }
19
+ context "when index only" do
20
+ subject { LogStash::Outputs::ElasticSearch.new(settings) }
21
+
22
+ before do
23
+ subject.register
37
24
  end
38
25
 
39
- it "should default to ES version" do
40
- subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
41
- r = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
42
- expect(r["_version"]).to eq(1)
43
- expect(r["_source"]["message"]).to eq('foo')
44
- subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
45
- r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
46
- expect(r2["_version"]).to eq(2)
47
- expect(r2["_source"]["message"]).to eq('foobar')
48
- end
49
- end
26
+ describe "unversioned output" do
27
+ let(:settings) do
28
+ {
29
+ "manage_template" => true,
30
+ "index" => "logstash-index",
31
+ "template_overwrite" => true,
32
+ "hosts" => get_host_port(),
33
+ "action" => "index",
34
+ "script_lang" => "groovy",
35
+ "document_id" => "%{my_id}"
36
+ }
37
+ end
50
38
 
51
- describe "versioned output" do
52
- let(:settings) do
53
- {
54
- "manage_template" => true,
55
- "index" => "logstash-index",
56
- "template_overwrite" => true,
57
- "hosts" => get_host_port(),
58
- "action" => "index",
59
- "script_lang" => "groovy",
60
- "document_id" => "%{my_id}",
61
- "version" => "%{my_version}",
62
- "version_type" => "external",
63
- }
39
+ it "should default to ES version" do
40
+ subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
41
+ r = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
42
+ expect(r["_version"]).to eq(1)
43
+ expect(r["_source"]["message"]).to eq('foo')
44
+ subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
45
+ r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
46
+ expect(r2["_version"]).to eq(2)
47
+ expect(r2["_source"]["message"]).to eq('foobar')
48
+ end
64
49
  end
65
50
 
66
- it "should respect the external version" do
67
- id = "ev1"
68
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
69
- r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
70
- expect(r["_version"]).to eq(99)
71
- expect(r["_source"]["message"]).to eq('foo')
72
- end
51
+ describe "versioned output" do
52
+ let(:settings) do
53
+ {
54
+ "manage_template" => true,
55
+ "index" => "logstash-index",
56
+ "template_overwrite" => true,
57
+ "hosts" => get_host_port(),
58
+ "action" => "index",
59
+ "script_lang" => "groovy",
60
+ "document_id" => "%{my_id}",
61
+ "version" => "%{my_version}",
62
+ "version_type" => "external",
63
+ }
64
+ end
73
65
 
74
- it "should ignore non-monotonic external version updates" do
75
- id = "ev2"
76
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
77
- r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
78
- expect(r["_version"]).to eq(99)
79
- expect(r["_source"]["message"]).to eq('foo')
66
+ it "should respect the external version" do
67
+ id = "ev1"
68
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
69
+ r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
70
+ expect(r["_version"]).to eq(99)
71
+ expect(r["_source"]["message"]).to eq('foo')
72
+ end
80
73
 
81
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
82
- r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
83
- expect(r2["_version"]).to eq(99)
84
- expect(r2["_source"]["message"]).to eq('foo')
85
- end
74
+ it "should ignore non-monotonic external version updates" do
75
+ id = "ev2"
76
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
77
+ r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
78
+ expect(r["_version"]).to eq(99)
79
+ expect(r["_source"]["message"]).to eq('foo')
80
+
81
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
82
+ r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
83
+ expect(r2["_version"]).to eq(99)
84
+ expect(r2["_source"]["message"]).to eq('foo')
85
+ end
86
86
 
87
- it "should commit monotonic external version updates" do
88
- id = "ev3"
89
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
90
- r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
91
- expect(r["_version"]).to eq(99)
92
- expect(r["_source"]["message"]).to eq('foo')
87
+ it "should commit monotonic external version updates" do
88
+ id = "ev3"
89
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
90
+ r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
91
+ expect(r["_version"]).to eq(99)
92
+ expect(r["_source"]["message"]).to eq('foo')
93
93
 
94
- subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
95
- r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
96
- expect(r2["_version"]).to eq(100)
97
- expect(r2["_source"]["message"]).to eq('foo')
94
+ subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
95
+ r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
96
+ expect(r2["_version"]).to eq(100)
97
+ expect(r2["_source"]["message"]).to eq('foo')
98
+ end
98
99
  end
99
100
  end
100
101
  end