logstash-output-elasticsearch 11.0.0-java → 11.0.4-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/docs/index.asciidoc +13 -13
- data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +1 -1
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +2 -28
- data/lib/logstash/outputs/elasticsearch/ilm.rb +2 -33
- data/lib/logstash/outputs/elasticsearch/license_checker.rb +12 -6
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +1 -1
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json +1 -0
- data/lib/logstash/outputs/elasticsearch.rb +18 -15
- data/lib/logstash/plugin_mixins/elasticsearch/common.rb +2 -1
- data/logstash-output-elasticsearch.gemspec +2 -2
- data/spec/es_spec_helper.rb +4 -6
- data/spec/fixtures/_nodes/{5x_6x.json → 6x.json} +5 -5
- data/spec/integration/outputs/compressed_indexing_spec.rb +47 -46
- data/spec/integration/outputs/delete_spec.rb +49 -51
- data/spec/integration/outputs/ilm_spec.rb +230 -246
- data/spec/integration/outputs/index_spec.rb +5 -2
- data/spec/integration/outputs/index_version_spec.rb +78 -82
- data/spec/integration/outputs/ingest_pipeline_spec.rb +58 -60
- data/spec/integration/outputs/painless_update_spec.rb +74 -164
- data/spec/integration/outputs/parent_spec.rb +67 -75
- data/spec/integration/outputs/retry_spec.rb +2 -2
- data/spec/integration/outputs/sniffer_spec.rb +15 -53
- data/spec/integration/outputs/templates_spec.rb +79 -81
- data/spec/integration/outputs/update_spec.rb +99 -101
- data/spec/spec_helper.rb +1 -5
- data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +0 -14
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +30 -37
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +9 -9
- data/spec/unit/outputs/elasticsearch_spec.rb +54 -18
- metadata +8 -22
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json +0 -95
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json +0 -46
- data/spec/fixtures/_nodes/2x_1x.json +0 -27
- data/spec/fixtures/scripts/groovy/scripted_update.groovy +0 -2
- data/spec/fixtures/scripts/groovy/scripted_update_nested.groovy +0 -2
- data/spec/fixtures/scripts/groovy/scripted_upsert.groovy +0 -2
- data/spec/integration/outputs/groovy_update_spec.rb +0 -150
- data/spec/integration/outputs/templates_5x_spec.rb +0 -98
@@ -80,8 +80,11 @@ describe "indexing" do
|
|
80
80
|
response = http_client.get("#{index_url}/_search?q=*&size=1000")
|
81
81
|
result = LogStash::Json.load(response.body)
|
82
82
|
result["hits"]["hits"].each do |doc|
|
83
|
-
|
84
|
-
|
83
|
+
if ESHelper.es_version_satisfies?("< 8")
|
84
|
+
expect(doc["_type"]).to eq(type)
|
85
|
+
else
|
86
|
+
expect(doc).not_to include("_type")
|
87
|
+
end
|
85
88
|
expect(doc["_index"]).to eq(index)
|
86
89
|
end
|
87
90
|
end
|
@@ -1,101 +1,97 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
require "logstash/outputs/elasticsearch"
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
require "logstash/outputs/elasticsearch"
|
4
|
+
describe "Versioned indexing", :integration => true do
|
5
|
+
require "logstash/outputs/elasticsearch"
|
7
6
|
|
8
|
-
|
7
|
+
let(:es) { get_client }
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
9
|
+
before :each do
|
10
|
+
# Delete all templates first.
|
11
|
+
# Clean ES of data before we start.
|
12
|
+
es.indices.delete_template(:name => "*")
|
13
|
+
# This can fail if there are no indexes, ignore failure.
|
14
|
+
es.indices.delete(:index => "*") rescue nil
|
15
|
+
es.indices.refresh
|
16
|
+
end
|
18
17
|
|
19
|
-
|
20
|
-
|
18
|
+
context "when index only" do
|
19
|
+
subject { LogStash::Outputs::ElasticSearch.new(settings) }
|
20
|
+
|
21
|
+
before do
|
22
|
+
subject.register
|
23
|
+
end
|
21
24
|
|
22
|
-
|
23
|
-
|
25
|
+
describe "unversioned output" do
|
26
|
+
let(:settings) do
|
27
|
+
{
|
28
|
+
"manage_template" => true,
|
29
|
+
"index" => "logstash-index",
|
30
|
+
"template_overwrite" => true,
|
31
|
+
"hosts" => get_host_port(),
|
32
|
+
"action" => "index",
|
33
|
+
"document_id" => "%{my_id}"
|
34
|
+
}
|
24
35
|
end
|
25
36
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
end
|
37
|
+
it "should default to ES version" do
|
38
|
+
subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
|
39
|
+
r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
|
40
|
+
expect(r["_version"]).to eq(1)
|
41
|
+
expect(r["_source"]["message"]).to eq('foo')
|
42
|
+
subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
|
43
|
+
r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
|
44
|
+
expect(r2["_version"]).to eq(2)
|
45
|
+
expect(r2["_source"]["message"]).to eq('foobar')
|
46
|
+
end
|
47
|
+
end
|
38
48
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
+
describe "versioned output" do
|
50
|
+
let(:settings) do
|
51
|
+
{
|
52
|
+
"manage_template" => true,
|
53
|
+
"index" => "logstash-index",
|
54
|
+
"template_overwrite" => true,
|
55
|
+
"hosts" => get_host_port(),
|
56
|
+
"action" => "index",
|
57
|
+
"document_id" => "%{my_id}",
|
58
|
+
"version" => "%{my_version}",
|
59
|
+
"version_type" => "external",
|
60
|
+
}
|
49
61
|
end
|
50
62
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
"action" => "index",
|
59
|
-
"script_lang" => "groovy",
|
60
|
-
"document_id" => "%{my_id}",
|
61
|
-
"version" => "%{my_version}",
|
62
|
-
"version_type" => "external",
|
63
|
-
}
|
64
|
-
end
|
65
|
-
|
66
|
-
it "should respect the external version" do
|
67
|
-
id = "ev1"
|
68
|
-
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
|
69
|
-
r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
|
70
|
-
expect(r["_version"]).to eq(99)
|
71
|
-
expect(r["_source"]["message"]).to eq('foo')
|
72
|
-
end
|
63
|
+
it "should respect the external version" do
|
64
|
+
id = "ev1"
|
65
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
|
66
|
+
r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
|
67
|
+
expect(r["_version"]).to eq(99)
|
68
|
+
expect(r["_source"]["message"]).to eq('foo')
|
69
|
+
end
|
73
70
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
71
|
+
it "should ignore non-monotonic external version updates" do
|
72
|
+
id = "ev2"
|
73
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
|
74
|
+
r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
|
75
|
+
expect(r["_version"]).to eq(99)
|
76
|
+
expect(r["_source"]["message"]).to eq('foo')
|
80
77
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
78
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
|
79
|
+
r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
|
80
|
+
expect(r2["_version"]).to eq(99)
|
81
|
+
expect(r2["_source"]["message"]).to eq('foo')
|
82
|
+
end
|
86
83
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
84
|
+
it "should commit monotonic external version updates" do
|
85
|
+
id = "ev3"
|
86
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
|
87
|
+
r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
|
88
|
+
expect(r["_version"]).to eq(99)
|
89
|
+
expect(r["_source"]["message"]).to eq('foo')
|
93
90
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
end
|
91
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
|
92
|
+
r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
|
93
|
+
expect(r2["_version"]).to eq(100)
|
94
|
+
expect(r2["_source"]["message"]).to eq('foo')
|
99
95
|
end
|
100
96
|
end
|
101
97
|
end
|
@@ -1,76 +1,74 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
end
|
3
|
+
describe "Ingest pipeline execution behavior", :integration => true do
|
4
|
+
subject! do
|
5
|
+
require "logstash/outputs/elasticsearch"
|
6
|
+
settings = {
|
7
|
+
"hosts" => "#{get_host_port()}",
|
8
|
+
"pipeline" => "apache-logs",
|
9
|
+
"data_stream" => 'false'
|
10
|
+
}
|
11
|
+
next LogStash::Outputs::ElasticSearch.new(settings)
|
12
|
+
end
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
}
|
14
|
+
let(:http_client) { Manticore::Client.new }
|
15
|
+
let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
|
16
|
+
let(:apache_logs_pipeline) { '
|
17
|
+
{
|
18
|
+
"description" : "Pipeline to parse Apache logs",
|
19
|
+
"processors" : [
|
20
|
+
{
|
21
|
+
"grok": {
|
22
|
+
"field": "message",
|
23
|
+
"patterns": ["%{COMBINEDAPACHELOG}"]
|
26
24
|
}
|
27
|
-
|
28
|
-
|
29
|
-
|
25
|
+
}
|
26
|
+
]
|
27
|
+
}'
|
28
|
+
}
|
30
29
|
|
31
|
-
|
32
|
-
|
33
|
-
|
30
|
+
before :each do
|
31
|
+
# Delete all templates first.
|
32
|
+
require "elasticsearch"
|
34
33
|
|
35
|
-
|
36
|
-
|
37
|
-
|
34
|
+
# Clean ES of data before we start.
|
35
|
+
@es = get_client
|
36
|
+
@es.indices.delete_template(:name => "*")
|
38
37
|
|
39
|
-
|
40
|
-
|
38
|
+
# This can fail if there are no indexes, ignore failure.
|
39
|
+
@es.indices.delete(:index => "*") rescue nil
|
41
40
|
|
42
|
-
|
43
|
-
|
41
|
+
# delete existing ingest pipeline
|
42
|
+
http_client.delete(ingest_url).call
|
44
43
|
|
45
|
-
|
46
|
-
|
44
|
+
# register pipeline
|
45
|
+
http_client.put(ingest_url, :body => apache_logs_pipeline, :headers => {"Content-Type" => "application/json" }).call
|
47
46
|
|
48
|
-
|
49
|
-
|
47
|
+
#TODO: Use esclient
|
48
|
+
#@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
|
50
49
|
|
51
|
-
|
52
|
-
|
53
|
-
|
50
|
+
subject.register
|
51
|
+
subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
|
52
|
+
@es.indices.refresh
|
54
53
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
end
|
54
|
+
#Wait or fail until everything's indexed.
|
55
|
+
Stud::try(10.times) do
|
56
|
+
r = @es.search(index: 'logstash-*')
|
57
|
+
expect(r).to have_hits(1)
|
58
|
+
sleep(0.1)
|
61
59
|
end
|
60
|
+
end
|
62
61
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
end
|
62
|
+
it "indexes using the proper pipeline" do
|
63
|
+
results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
|
64
|
+
expect(results).to have_hits(1)
|
65
|
+
expect(results["hits"]["hits"][0]["_source"]["response"]).to eq("200")
|
66
|
+
expect(results["hits"]["hits"][0]["_source"]["bytes"]).to eq("182")
|
67
|
+
expect(results["hits"]["hits"][0]["_source"]["verb"]).to eq("GET")
|
68
|
+
expect(results["hits"]["hits"][0]["_source"]["request"]).to eq("/scripts/netcat-webserver")
|
69
|
+
expect(results["hits"]["hits"][0]["_source"]["auth"]).to eq("-")
|
70
|
+
expect(results["hits"]["hits"][0]["_source"]["ident"]).to eq("-")
|
71
|
+
expect(results["hits"]["hits"][0]["_source"]["clientip"]).to eq("183.60.215.50")
|
72
|
+
expect(results["hits"]["hits"][0]["_source"]["junkfieldaaaa"]).to eq(nil)
|
75
73
|
end
|
76
74
|
end
|
@@ -1,188 +1,98 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
settings.merge!({"script_lang" => "painless"})
|
17
|
-
end
|
18
|
-
LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
|
19
|
-
end
|
20
|
-
|
21
|
-
before :each do
|
22
|
-
@es = get_client
|
23
|
-
# Delete all templates first.
|
24
|
-
# Clean ES of data before we start.
|
25
|
-
@es.indices.delete_template(:name => "*")
|
26
|
-
# This can fail if there are no indexes, ignore failure.
|
27
|
-
@es.indices.delete(:index => "*") rescue nil
|
28
|
-
@es.index(
|
29
|
-
:index => 'logstash-update',
|
30
|
-
:type => doc_type,
|
31
|
-
:id => "123",
|
32
|
-
:body => { :message => 'Test', :counter => 1 }
|
33
|
-
)
|
34
|
-
@es.indices.refresh
|
35
|
-
end
|
3
|
+
describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
|
4
|
+
require "logstash/outputs/elasticsearch"
|
5
|
+
|
6
|
+
def get_es_output( options={} )
|
7
|
+
settings = {
|
8
|
+
"manage_template" => true,
|
9
|
+
"index" => "logstash-update",
|
10
|
+
"template_overwrite" => true,
|
11
|
+
"hosts" => get_host_port(),
|
12
|
+
"action" => "update"
|
13
|
+
}
|
14
|
+
LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
|
15
|
+
end
|
36
16
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
17
|
+
before :each do
|
18
|
+
@es = get_client
|
19
|
+
# Delete all templates first.
|
20
|
+
# Clean ES of data before we start.
|
21
|
+
@es.indices.delete_template(:name => "*")
|
22
|
+
# This can fail if there are no indexes, ignore failure.
|
23
|
+
@es.indices.delete(:index => "*") rescue nil
|
24
|
+
@es.index(
|
25
|
+
:index => 'logstash-update',
|
26
|
+
:type => doc_type,
|
27
|
+
:id => "123",
|
28
|
+
:body => { :message => 'Test', :counter => 1 }
|
29
|
+
)
|
30
|
+
@es.indices.refresh
|
31
|
+
end
|
47
32
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
53
|
-
expect(r["_source"]["counter"]).to eq(4)
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
33
|
+
context "scripted updates" do
|
34
|
+
context 'with an indexed script' do
|
35
|
+
it "should increment a counter with event/doc 'count' variable with indexed script" do
|
36
|
+
@es.perform_request(:put, "_scripts/indexed_update", {}, {"script" => {"source" => "ctx._source.counter += params.event.count", "lang" => "painless"}})
|
57
37
|
|
58
|
-
|
59
|
-
subject = get_es_output({
|
38
|
+
plugin_parameters = {
|
60
39
|
'document_id' => "123",
|
61
|
-
'script' => '
|
62
|
-
'script_type' => '
|
63
|
-
}
|
64
|
-
subject.register
|
65
|
-
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
66
|
-
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
67
|
-
expect(r["_source"]["counter"]).to eq(4)
|
68
|
-
end
|
40
|
+
'script' => 'indexed_update',
|
41
|
+
'script_type' => 'indexed'
|
42
|
+
}
|
69
43
|
|
70
|
-
|
71
|
-
subject = get_es_output({
|
72
|
-
'document_id' => "123",
|
73
|
-
'doc_as_upsert' => true,
|
74
|
-
'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
|
75
|
-
'script_type' => 'inline'
|
76
|
-
})
|
77
|
-
subject.register
|
78
|
-
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
79
|
-
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
80
|
-
expect(r["_source"]["counter"]).to eq(4)
|
81
|
-
end
|
44
|
+
plugin_parameters.merge!('script_lang' => '')
|
82
45
|
|
83
|
-
|
84
|
-
subject = get_es_output({
|
85
|
-
'document_id' => "456",
|
86
|
-
'doc_as_upsert' => true,
|
87
|
-
'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
|
88
|
-
'script_type' => 'inline'
|
89
|
-
})
|
46
|
+
subject = get_es_output(plugin_parameters)
|
90
47
|
subject.register
|
91
|
-
subject.multi_receive([LogStash::Event.new("
|
92
|
-
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "
|
93
|
-
expect(r["_source"]["counter"]).to eq(
|
48
|
+
subject.multi_receive([LogStash::Event.new("count" => 4 )])
|
49
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
50
|
+
expect(r["_source"]["counter"]).to eq(5)
|
94
51
|
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context "when update with upsert" do
|
56
|
+
it "should create new documents with provided upsert" do
|
57
|
+
subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
58
|
+
subject.register
|
59
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
60
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
61
|
+
expect(r["_source"]["message"]).to eq('upsert message')
|
62
|
+
end
|
95
63
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
plugin_parameters = {
|
105
|
-
'document_id' => "123",
|
106
|
-
'script' => 'indexed_update',
|
107
|
-
'script_type' => 'indexed'
|
108
|
-
}
|
109
|
-
|
110
|
-
if ESHelper.es_version_satisfies?('>= 6.0.0')
|
111
|
-
plugin_parameters.merge!('script_lang' => '')
|
112
|
-
end
|
64
|
+
it "should create new documents with event/doc as upsert" do
|
65
|
+
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
|
66
|
+
subject.register
|
67
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
68
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
69
|
+
expect(r["_source"]["message"]).to eq('sample message here')
|
70
|
+
end
|
113
71
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
end
|
120
|
-
end
|
121
|
-
end
|
72
|
+
it "should fail on documents with event/doc as upsert at external version" do
|
73
|
+
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
|
74
|
+
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
75
|
+
end
|
76
|
+
end
|
122
77
|
|
123
|
-
|
124
|
-
|
125
|
-
|
78
|
+
context "updates with scripted upsert" do
|
79
|
+
context 'with an inline script' do
|
80
|
+
it "should create new documents with upsert content" do
|
81
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
|
126
82
|
subject.register
|
83
|
+
|
127
84
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
128
85
|
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
129
86
|
expect(r["_source"]["message"]).to eq('upsert message')
|
130
87
|
end
|
131
88
|
|
132
|
-
it "should create new documents with event/doc as
|
133
|
-
subject = get_es_output({ 'document_id' => "456", '
|
89
|
+
it "should create new documents with event/doc as script params" do
|
90
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
|
134
91
|
subject.register
|
135
|
-
subject.multi_receive([LogStash::Event.new("
|
92
|
+
subject.multi_receive([LogStash::Event.new("counter" => 1)])
|
93
|
+
@es.indices.refresh
|
136
94
|
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
137
|
-
expect(r["_source"]["
|
138
|
-
end
|
139
|
-
|
140
|
-
it "should fail on documents with event/doc as upsert at external version" do
|
141
|
-
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
|
142
|
-
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
143
|
-
end
|
144
|
-
end
|
145
|
-
|
146
|
-
context "updates with scripted upsert" do
|
147
|
-
if ESHelper.es_version_satisfies?('<6')
|
148
|
-
context 'with file based scripts' do
|
149
|
-
it "should create new documents with upsert content" do
|
150
|
-
subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
|
151
|
-
subject.register
|
152
|
-
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
153
|
-
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
154
|
-
expect(r["_source"]["message"]).to eq('upsert message')
|
155
|
-
end
|
156
|
-
|
157
|
-
it "should create new documents with event/doc as script params" do
|
158
|
-
subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
|
159
|
-
subject.register
|
160
|
-
subject.multi_receive([LogStash::Event.new("counter" => 1)])
|
161
|
-
@es.indices.refresh
|
162
|
-
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
163
|
-
expect(r["_source"]["counter"]).to eq(1)
|
164
|
-
end
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
|
-
context 'with an inline script' do
|
169
|
-
it "should create new documents with upsert content" do
|
170
|
-
subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
|
171
|
-
subject.register
|
172
|
-
|
173
|
-
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
174
|
-
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
175
|
-
expect(r["_source"]["message"]).to eq('upsert message')
|
176
|
-
end
|
177
|
-
|
178
|
-
it "should create new documents with event/doc as script params" do
|
179
|
-
subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
|
180
|
-
subject.register
|
181
|
-
subject.multi_receive([LogStash::Event.new("counter" => 1)])
|
182
|
-
@es.indices.refresh
|
183
|
-
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
184
|
-
expect(r["_source"]["counter"]).to eq(1)
|
185
|
-
end
|
95
|
+
expect(r["_source"]["counter"]).to eq(1)
|
186
96
|
end
|
187
97
|
end
|
188
98
|
end
|