logstash-output-elasticsearch 7.4.0-java → 7.4.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +3 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +6 -3
- data/lib/logstash/outputs/elasticsearch/http_client.rb +7 -9
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +31 -12
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +4 -6
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/es_spec_helper.rb +6 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +46 -44
- data/spec/integration/outputs/delete_spec.rb +51 -49
- data/spec/integration/outputs/groovy_update_spec.rb +131 -129
- data/spec/integration/outputs/index_version_spec.rb +82 -81
- data/spec/integration/outputs/ingest_pipeline_spec.rb +51 -49
- data/spec/integration/outputs/painless_update_spec.rb +132 -130
- data/spec/integration/outputs/parent_spec.rb +56 -54
- data/spec/integration/outputs/sniffer_spec.rb +5 -2
- data/spec/integration/outputs/templates_5x_spec.rb +81 -82
- data/spec/integration/outputs/templates_spec.rb +81 -81
- data/spec/integration/outputs/update_spec.rb +101 -99
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +3 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +1 -1
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +13 -25
- metadata +4 -6
@@ -1,148 +1,150 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
|
3
|
-
|
4
|
-
|
3
|
+
if ESHelper.es_version_satisfies?(">= 2")
|
4
|
+
describe "Update actions using groovy scripts", :integration => true, :update_tests => 'groovy' do
|
5
|
+
require "logstash/outputs/elasticsearch"
|
5
6
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
end
|
17
|
-
|
18
|
-
before :each do
|
19
|
-
@es = get_client
|
20
|
-
# Delete all templates first.
|
21
|
-
# Clean ES of data before we start.
|
22
|
-
@es.indices.delete_template(:name => "*")
|
23
|
-
# This can fail if there are no indexes, ignore failure.
|
24
|
-
@es.indices.delete(:index => "*") rescue nil
|
25
|
-
@es.index(
|
26
|
-
:index => 'logstash-update',
|
27
|
-
:type => 'logs',
|
28
|
-
:id => "123",
|
29
|
-
:body => { :message => 'Test', :counter => 1 }
|
30
|
-
)
|
31
|
-
@es.indices.refresh
|
32
|
-
end
|
33
|
-
|
34
|
-
context "scripted updates" do
|
35
|
-
it "should increment a counter with event/doc 'count' variable" do
|
36
|
-
subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
|
37
|
-
subject.register
|
38
|
-
subject.multi_receive([LogStash::Event.new("count" => 2)])
|
39
|
-
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
40
|
-
insist { r["_source"]["counter"] } == 3
|
7
|
+
def get_es_output( options={} )
|
8
|
+
settings = {
|
9
|
+
"manage_template" => true,
|
10
|
+
"index" => "logstash-update",
|
11
|
+
"template_overwrite" => true,
|
12
|
+
"hosts" => get_host_port(),
|
13
|
+
"action" => "update",
|
14
|
+
"script_lang" => "groovy"
|
15
|
+
}
|
16
|
+
LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
|
41
17
|
end
|
42
18
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
19
|
+
before :each do
|
20
|
+
@es = get_client
|
21
|
+
# Delete all templates first.
|
22
|
+
# Clean ES of data before we start.
|
23
|
+
@es.indices.delete_template(:name => "*")
|
24
|
+
# This can fail if there are no indexes, ignore failure.
|
25
|
+
@es.indices.delete(:index => "*") rescue nil
|
26
|
+
@es.index(
|
27
|
+
:index => 'logstash-update',
|
28
|
+
:type => 'logs',
|
29
|
+
:id => "123",
|
30
|
+
:body => { :message => 'Test', :counter => 1 }
|
31
|
+
)
|
32
|
+
@es.indices.refresh
|
49
33
|
end
|
50
34
|
|
51
|
-
|
52
|
-
|
53
|
-
'document_id' => "123",
|
54
|
-
|
55
|
-
|
56
|
-
'
|
57
|
-
|
58
|
-
|
59
|
-
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
60
|
-
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
61
|
-
insist { r["_source"]["counter"] } == 4
|
62
|
-
end
|
35
|
+
context "scripted updates" do
|
36
|
+
it "should increment a counter with event/doc 'count' variable" do
|
37
|
+
subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
|
38
|
+
subject.register
|
39
|
+
subject.multi_receive([LogStash::Event.new("count" => 2)])
|
40
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
41
|
+
insist { r["_source"]["counter"] } == 3
|
42
|
+
end
|
63
43
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
})
|
72
|
-
subject.register
|
73
|
-
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
74
|
-
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
75
|
-
insist { r["_source"]["counter"] } == 4
|
76
|
-
end
|
44
|
+
it "should increment a counter with event/doc '[data][count]' nested variable" do
|
45
|
+
subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
|
46
|
+
subject.register
|
47
|
+
subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
|
48
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
49
|
+
insist { r["_source"]["counter"] } == 4
|
50
|
+
end
|
77
51
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
end
|
52
|
+
it "should increment a counter with event/doc 'count' variable with inline script" do
|
53
|
+
subject = get_es_output({
|
54
|
+
'document_id' => "123",
|
55
|
+
'script' => 'ctx._source.counter += event["counter"]',
|
56
|
+
'script_lang' => 'groovy',
|
57
|
+
'script_type' => 'inline'
|
58
|
+
})
|
59
|
+
subject.register
|
60
|
+
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
61
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
62
|
+
insist { r["_source"]["counter"] } == 4
|
63
|
+
end
|
91
64
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
end
|
106
|
-
|
107
|
-
context "when update with upsert" do
|
108
|
-
it "should create new documents with provided upsert" do
|
109
|
-
subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
110
|
-
subject.register
|
111
|
-
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
112
|
-
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
113
|
-
insist { r["_source"]["message"] } == 'upsert message'
|
114
|
-
end
|
65
|
+
it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
|
66
|
+
subject = get_es_output({
|
67
|
+
'document_id' => "123",
|
68
|
+
'doc_as_upsert' => true,
|
69
|
+
'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
|
70
|
+
'script_lang' => 'groovy',
|
71
|
+
'script_type' => 'inline'
|
72
|
+
})
|
73
|
+
subject.register
|
74
|
+
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
75
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
76
|
+
insist { r["_source"]["counter"] } == 4
|
77
|
+
end
|
115
78
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
79
|
+
it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
|
80
|
+
subject = get_es_output({
|
81
|
+
'document_id' => "456",
|
82
|
+
'doc_as_upsert' => true,
|
83
|
+
'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
|
84
|
+
'script_lang' => 'groovy',
|
85
|
+
'script_type' => 'inline'
|
86
|
+
})
|
87
|
+
subject.register
|
88
|
+
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
89
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
90
|
+
insist { r["_source"]["counter"] } == 3
|
91
|
+
end
|
123
92
|
|
124
|
-
|
125
|
-
|
126
|
-
|
93
|
+
it "should increment a counter with event/doc 'count' variable with indexed script" do
|
94
|
+
@es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
|
95
|
+
subject = get_es_output({
|
96
|
+
'document_id' => "123",
|
97
|
+
'script' => 'indexed_update',
|
98
|
+
'script_lang' => 'groovy',
|
99
|
+
'script_type' => 'indexed'
|
100
|
+
})
|
101
|
+
subject.register
|
102
|
+
subject.multi_receive([LogStash::Event.new("count" => 4 )])
|
103
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
104
|
+
insist { r["_source"]["counter"] } == 5
|
105
|
+
end
|
127
106
|
end
|
128
|
-
end
|
129
107
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
108
|
+
context "when update with upsert" do
|
109
|
+
it "should create new documents with provided upsert" do
|
110
|
+
subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
111
|
+
subject.register
|
112
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
113
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
114
|
+
insist { r["_source"]["message"] } == 'upsert message'
|
115
|
+
end
|
116
|
+
|
117
|
+
it "should create new documents with event/doc as upsert" do
|
118
|
+
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
|
119
|
+
subject.register
|
120
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
121
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
122
|
+
insist { r["_source"]["message"] } == 'sample message here'
|
123
|
+
end
|
124
|
+
|
125
|
+
it "should fail on documents with event/doc as upsert at external version" do
|
126
|
+
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
|
127
|
+
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
128
|
+
end
|
137
129
|
end
|
138
130
|
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
131
|
+
context "updates with scripted upsert" do
|
132
|
+
it "should create new documents with upsert content" do
|
133
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
|
134
|
+
subject.register
|
135
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
136
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
137
|
+
insist { r["_source"]["message"] } == 'upsert message'
|
138
|
+
end
|
139
|
+
|
140
|
+
it "should create new documents with event/doc as script params" do
|
141
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
|
142
|
+
subject.register
|
143
|
+
subject.multi_receive([LogStash::Event.new("counter" => 1)])
|
144
|
+
@es.indices.refresh
|
145
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
146
|
+
insist { r["_source"]["counter"] } == 1
|
147
|
+
end
|
146
148
|
end
|
147
149
|
end
|
148
150
|
end
|
@@ -1,100 +1,101 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
require "logstash/outputs/elasticsearch"
|
3
3
|
|
4
|
+
if ESHelper.es_version_satisfies?(">= 2")
|
5
|
+
describe "Versioned indexing", :integration => true do
|
6
|
+
require "logstash/outputs/elasticsearch"
|
4
7
|
|
5
|
-
|
6
|
-
require "logstash/outputs/elasticsearch"
|
8
|
+
let(:es) { get_client }
|
7
9
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
es.indices.delete(:index => "*") rescue nil
|
16
|
-
es.indices.refresh
|
17
|
-
end
|
18
|
-
|
19
|
-
context "when index only" do
|
20
|
-
subject { LogStash::Outputs::ElasticSearch.new(settings) }
|
21
|
-
|
22
|
-
before do
|
23
|
-
subject.register
|
10
|
+
before :each do
|
11
|
+
# Delete all templates first.
|
12
|
+
# Clean ES of data before we start.
|
13
|
+
es.indices.delete_template(:name => "*")
|
14
|
+
# This can fail if there are no indexes, ignore failure.
|
15
|
+
es.indices.delete(:index => "*") rescue nil
|
16
|
+
es.indices.refresh
|
24
17
|
end
|
25
18
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
"template_overwrite" => true,
|
32
|
-
"hosts" => get_host_port(),
|
33
|
-
"action" => "index",
|
34
|
-
"script_lang" => "groovy",
|
35
|
-
"document_id" => "%{my_id}"
|
36
|
-
}
|
19
|
+
context "when index only" do
|
20
|
+
subject { LogStash::Outputs::ElasticSearch.new(settings) }
|
21
|
+
|
22
|
+
before do
|
23
|
+
subject.register
|
37
24
|
end
|
38
25
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
26
|
+
describe "unversioned output" do
|
27
|
+
let(:settings) do
|
28
|
+
{
|
29
|
+
"manage_template" => true,
|
30
|
+
"index" => "logstash-index",
|
31
|
+
"template_overwrite" => true,
|
32
|
+
"hosts" => get_host_port(),
|
33
|
+
"action" => "index",
|
34
|
+
"script_lang" => "groovy",
|
35
|
+
"document_id" => "%{my_id}"
|
36
|
+
}
|
37
|
+
end
|
50
38
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
"
|
55
|
-
"
|
56
|
-
"
|
57
|
-
"
|
58
|
-
"
|
59
|
-
"
|
60
|
-
|
61
|
-
"version" => "%{my_version}",
|
62
|
-
"version_type" => "external",
|
63
|
-
}
|
39
|
+
it "should default to ES version" do
|
40
|
+
subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
|
41
|
+
r = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
|
42
|
+
expect(r["_version"]).to eq(1)
|
43
|
+
expect(r["_source"]["message"]).to eq('foo')
|
44
|
+
subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
|
45
|
+
r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
|
46
|
+
expect(r2["_version"]).to eq(2)
|
47
|
+
expect(r2["_source"]["message"]).to eq('foobar')
|
48
|
+
end
|
64
49
|
end
|
65
50
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
51
|
+
describe "versioned output" do
|
52
|
+
let(:settings) do
|
53
|
+
{
|
54
|
+
"manage_template" => true,
|
55
|
+
"index" => "logstash-index",
|
56
|
+
"template_overwrite" => true,
|
57
|
+
"hosts" => get_host_port(),
|
58
|
+
"action" => "index",
|
59
|
+
"script_lang" => "groovy",
|
60
|
+
"document_id" => "%{my_id}",
|
61
|
+
"version" => "%{my_version}",
|
62
|
+
"version_type" => "external",
|
63
|
+
}
|
64
|
+
end
|
73
65
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
66
|
+
it "should respect the external version" do
|
67
|
+
id = "ev1"
|
68
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
|
69
|
+
r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
|
70
|
+
expect(r["_version"]).to eq(99)
|
71
|
+
expect(r["_source"]["message"]).to eq('foo')
|
72
|
+
end
|
80
73
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
74
|
+
it "should ignore non-monotonic external version updates" do
|
75
|
+
id = "ev2"
|
76
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
|
77
|
+
r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
|
78
|
+
expect(r["_version"]).to eq(99)
|
79
|
+
expect(r["_source"]["message"]).to eq('foo')
|
80
|
+
|
81
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
|
82
|
+
r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
|
83
|
+
expect(r2["_version"]).to eq(99)
|
84
|
+
expect(r2["_source"]["message"]).to eq('foo')
|
85
|
+
end
|
86
86
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
87
|
+
it "should commit monotonic external version updates" do
|
88
|
+
id = "ev3"
|
89
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
|
90
|
+
r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
|
91
|
+
expect(r["_version"]).to eq(99)
|
92
|
+
expect(r["_source"]["message"]).to eq('foo')
|
93
93
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
94
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
|
95
|
+
r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
|
96
|
+
expect(r2["_version"]).to eq(100)
|
97
|
+
expect(r2["_source"]["message"]).to eq('foo')
|
98
|
+
end
|
98
99
|
end
|
99
100
|
end
|
100
101
|
end
|