logstash-output-elasticsearch 8.1.1-java → 8.2.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +6 -3
- data/lib/logstash/outputs/elasticsearch/http_client.rb +7 -9
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +31 -12
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +4 -6
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/es_spec_helper.rb +6 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +46 -44
- data/spec/integration/outputs/delete_spec.rb +51 -49
- data/spec/integration/outputs/groovy_update_spec.rb +131 -129
- data/spec/integration/outputs/index_version_spec.rb +82 -81
- data/spec/integration/outputs/ingest_pipeline_spec.rb +51 -49
- data/spec/integration/outputs/painless_update_spec.rb +161 -130
- data/spec/integration/outputs/parent_spec.rb +133 -123
- data/spec/integration/outputs/sniffer_spec.rb +5 -2
- data/spec/integration/outputs/templates_5x_spec.rb +81 -82
- data/spec/integration/outputs/templates_spec.rb +81 -81
- data/spec/integration/outputs/update_spec.rb +101 -99
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +3 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +1 -1
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +13 -25
- metadata +2 -2
@@ -1,18 +1,19 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
3
|
+
if ESHelper.es_version_satisfies?(">= 5")
|
4
|
+
describe "Ingest pipeline execution behavior", :integration => true do
|
5
|
+
subject! do
|
6
|
+
require "logstash/outputs/elasticsearch"
|
7
|
+
settings = {
|
8
|
+
"hosts" => "#{get_host_port()}",
|
9
|
+
"pipeline" => "apache-logs"
|
10
|
+
}
|
11
|
+
next LogStash::Outputs::ElasticSearch.new(settings)
|
12
|
+
end
|
12
13
|
|
13
|
-
|
14
|
-
|
15
|
-
|
14
|
+
let(:ftw_client) { FTW::Agent.new }
|
15
|
+
let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
|
16
|
+
let(:apache_logs_pipeline) { '
|
16
17
|
{
|
17
18
|
"description" : "Pipeline to parse Apache logs",
|
18
19
|
"processors" : [
|
@@ -24,52 +25,53 @@ describe "Ingest pipeline execution behavior", :integration => true, :version_gr
|
|
24
25
|
}
|
25
26
|
]
|
26
27
|
}'
|
27
|
-
|
28
|
+
}
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
|
30
|
+
before :each do
|
31
|
+
# Delete all templates first.
|
32
|
+
require "elasticsearch"
|
32
33
|
|
33
|
-
|
34
|
-
|
35
|
-
|
34
|
+
# Clean ES of data before we start.
|
35
|
+
@es = get_client
|
36
|
+
@es.indices.delete_template(:name => "*")
|
36
37
|
|
37
|
-
|
38
|
-
|
38
|
+
# This can fail if there are no indexes, ignore failure.
|
39
|
+
@es.indices.delete(:index => "*") rescue nil
|
39
40
|
|
40
|
-
|
41
|
-
|
42
|
-
|
41
|
+
# delete existing ingest pipeline
|
42
|
+
req = ftw_client.delete(ingest_url)
|
43
|
+
ftw_client.execute(req)
|
43
44
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
45
|
+
# register pipeline
|
46
|
+
req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
|
47
|
+
req.headers["Content-Type"] = "application/json"
|
48
|
+
ftw_client.execute(req)
|
48
49
|
|
49
|
-
|
50
|
-
|
50
|
+
#TODO: Use esclient
|
51
|
+
#@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
|
51
52
|
|
52
|
-
|
53
|
-
|
54
|
-
|
53
|
+
subject.register
|
54
|
+
subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
|
55
|
+
@es.indices.refresh
|
55
56
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
57
|
+
#Wait or fail until everything's indexed.
|
58
|
+
Stud::try(20.times) do
|
59
|
+
r = @es.search
|
60
|
+
insist { r["hits"]["total"] } == 1
|
61
|
+
end
|
60
62
|
end
|
61
|
-
end
|
62
63
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
64
|
+
it "indexes using the proper pipeline" do
|
65
|
+
results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
|
66
|
+
insist { results["hits"]["total"] } == 1
|
67
|
+
insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
|
68
|
+
insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
|
69
|
+
insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
|
70
|
+
insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
|
71
|
+
insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
|
72
|
+
insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
|
73
|
+
insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
|
74
|
+
insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
|
75
|
+
end
|
74
76
|
end
|
75
77
|
end
|
@@ -1,148 +1,179 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
def get_es_output( options={} )
|
7
|
-
settings = {
|
8
|
-
"manage_template" => true,
|
9
|
-
"index" => "logstash-update",
|
10
|
-
"template_overwrite" => true,
|
11
|
-
"hosts" => get_host_port(),
|
12
|
-
"action" => "update",
|
13
|
-
"script_lang" => "painless"
|
14
|
-
}
|
15
|
-
LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
|
16
|
-
end
|
17
|
-
|
18
|
-
before :each do
|
19
|
-
@es = get_client
|
20
|
-
# Delete all templates first.
|
21
|
-
# Clean ES of data before we start.
|
22
|
-
@es.indices.delete_template(:name => "*")
|
23
|
-
# This can fail if there are no indexes, ignore failure.
|
24
|
-
@es.indices.delete(:index => "*") rescue nil
|
25
|
-
@es.index(
|
26
|
-
:index => 'logstash-update',
|
27
|
-
:type => 'logs',
|
28
|
-
:id => "123",
|
29
|
-
:body => { :message => 'Test', :counter => 1 }
|
30
|
-
)
|
31
|
-
@es.indices.refresh
|
32
|
-
end
|
3
|
+
if ESHelper.es_version_satisfies?(">= 5")
|
4
|
+
describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
|
5
|
+
require "logstash/outputs/elasticsearch"
|
33
6
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
7
|
+
def get_es_output( options={} )
|
8
|
+
settings = {
|
9
|
+
"manage_template" => true,
|
10
|
+
"index" => "logstash-update",
|
11
|
+
"template_overwrite" => true,
|
12
|
+
"hosts" => get_host_port(),
|
13
|
+
"action" => "update"
|
14
|
+
}
|
15
|
+
if ESHelper.es_version_satisfies?('<6')
|
16
|
+
settings.merge!({"script_lang" => "painless"})
|
17
|
+
end
|
18
|
+
LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
|
41
19
|
end
|
42
20
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
21
|
+
before :each do
|
22
|
+
@es = get_client
|
23
|
+
# Delete all templates first.
|
24
|
+
# Clean ES of data before we start.
|
25
|
+
@es.indices.delete_template(:name => "*")
|
26
|
+
# This can fail if there are no indexes, ignore failure.
|
27
|
+
@es.indices.delete(:index => "*") rescue nil
|
28
|
+
@es.index(
|
29
|
+
:index => 'logstash-update',
|
30
|
+
:type => 'logs',
|
31
|
+
:id => "123",
|
32
|
+
:body => { :message => 'Test', :counter => 1 }
|
33
|
+
)
|
34
|
+
@es.indices.refresh
|
49
35
|
end
|
50
36
|
|
51
|
-
|
52
|
-
|
53
|
-
'
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
insist { r["_source"]["counter"] } == 4
|
62
|
-
end
|
37
|
+
context "scripted updates" do
|
38
|
+
if ESHelper.es_version_satisfies?('<6')
|
39
|
+
context 'with file based scripts' do
|
40
|
+
it "should increment a counter with event/doc 'count' variable" do
|
41
|
+
subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
|
42
|
+
subject.register
|
43
|
+
subject.multi_receive([LogStash::Event.new("count" => 2)])
|
44
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
45
|
+
insist { r["_source"]["counter"] } == 3
|
46
|
+
end
|
63
47
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
74
|
-
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
75
|
-
insist { r["_source"]["counter"] } == 4
|
76
|
-
end
|
48
|
+
it "should increment a counter with event/doc '[data][count]' nested variable" do
|
49
|
+
subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
|
50
|
+
subject.register
|
51
|
+
subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
|
52
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
53
|
+
insist { r["_source"]["counter"] } == 4
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
77
57
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
insist { r["_source"]["counter"] } == 3
|
90
|
-
end
|
58
|
+
it "should increment a counter with event/doc 'count' variable with inline script" do
|
59
|
+
subject = get_es_output({
|
60
|
+
'document_id' => "123",
|
61
|
+
'script' => 'ctx._source.counter += params.event.counter',
|
62
|
+
'script_type' => 'inline'
|
63
|
+
})
|
64
|
+
subject.register
|
65
|
+
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
66
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
67
|
+
insist { r["_source"]["counter"] } == 4
|
68
|
+
end
|
91
69
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
context "when update with upsert" do
|
108
|
-
it "should create new documents with provided upsert" do
|
109
|
-
subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
110
|
-
subject.register
|
111
|
-
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
112
|
-
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
113
|
-
insist { r["_source"]["message"] } == 'upsert message'
|
114
|
-
end
|
70
|
+
it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
|
71
|
+
subject = get_es_output({
|
72
|
+
'document_id' => "123",
|
73
|
+
'doc_as_upsert' => true,
|
74
|
+
'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
|
75
|
+
'script_type' => 'inline'
|
76
|
+
})
|
77
|
+
subject.register
|
78
|
+
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
79
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
80
|
+
insist { r["_source"]["counter"] } == 4
|
81
|
+
end
|
115
82
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
83
|
+
it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
|
84
|
+
subject = get_es_output({
|
85
|
+
'document_id' => "456",
|
86
|
+
'doc_as_upsert' => true,
|
87
|
+
'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
|
88
|
+
'script_type' => 'inline'
|
89
|
+
})
|
90
|
+
subject.register
|
91
|
+
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
92
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
93
|
+
insist { r["_source"]["counter"] } == 3
|
94
|
+
end
|
123
95
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
96
|
+
if ESHelper.es_version_satisfies?('<6')
|
97
|
+
context 'with an indexed script' do
|
98
|
+
it "should increment a counter with event/doc 'count' variable with indexed script" do
|
99
|
+
@es.put_script lang: 'painless', id: 'indexed_update', body: { script: 'ctx._source.counter += params.event.count' }
|
100
|
+
subject = get_es_output({
|
101
|
+
'document_id' => "123",
|
102
|
+
'script' => 'indexed_update',
|
103
|
+
'script_type' => 'indexed'
|
104
|
+
})
|
105
|
+
subject.register
|
106
|
+
subject.multi_receive([LogStash::Event.new("count" => 4 )])
|
107
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
108
|
+
insist { r["_source"]["counter"] } == 5
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
context "when update with upsert" do
|
115
|
+
it "should create new documents with provided upsert" do
|
116
|
+
subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
117
|
+
subject.register
|
118
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
119
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
120
|
+
insist { r["_source"]["message"] } == 'upsert message'
|
121
|
+
end
|
122
|
+
|
123
|
+
it "should create new documents with event/doc as upsert" do
|
124
|
+
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
|
125
|
+
subject.register
|
126
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
127
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
128
|
+
insist { r["_source"]["message"] } == 'sample message here'
|
129
|
+
end
|
129
130
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
135
|
-
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
136
|
-
insist { r["_source"]["message"] } == 'upsert message'
|
131
|
+
it "should fail on documents with event/doc as upsert at external version" do
|
132
|
+
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
|
133
|
+
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
134
|
+
end
|
137
135
|
end
|
138
136
|
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
137
|
+
context "updates with scripted upsert" do
|
138
|
+
if ESHelper.es_version_satisfies?('<6')
|
139
|
+
context 'with file based scripts' do
|
140
|
+
it "should create new documents with upsert content" do
|
141
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
|
142
|
+
subject.register
|
143
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
144
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
145
|
+
insist { r["_source"]["message"] } == 'upsert message'
|
146
|
+
end
|
147
|
+
|
148
|
+
it "should create new documents with event/doc as script params" do
|
149
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
|
150
|
+
subject.register
|
151
|
+
subject.multi_receive([LogStash::Event.new("counter" => 1)])
|
152
|
+
@es.indices.refresh
|
153
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
154
|
+
insist { r["_source"]["counter"] } == 1
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
context 'with an inline script' do
|
160
|
+
it "should create new documents with upsert content" do
|
161
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
|
162
|
+
subject.register
|
163
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
164
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
165
|
+
insist { r["_source"]["message"] } == 'upsert message'
|
166
|
+
end
|
167
|
+
|
168
|
+
it "should create new documents with event/doc as script params" do
|
169
|
+
subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
|
170
|
+
subject.register
|
171
|
+
subject.multi_receive([LogStash::Event.new("counter" => 1)])
|
172
|
+
@es.indices.refresh
|
173
|
+
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
|
174
|
+
insist { r["_source"]["counter"] } == 1
|
175
|
+
end
|
176
|
+
end
|
146
177
|
end
|
147
178
|
end
|
148
179
|
end
|
@@ -1,159 +1,169 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
require "logstash/outputs/elasticsearch"
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
{
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
4
|
+
if ESHelper.es_version_satisfies?("<= 5.x")
|
5
|
+
context "when using elasticsearch 5.x and before", :integration => true do
|
6
|
+
shared_examples "a type based parent indexer" do
|
7
|
+
let(:index) { 10.times.collect { rand(10).to_s }.join("") }
|
8
|
+
let(:type) { 10.times.collect { rand(10).to_s }.join("") }
|
9
|
+
let(:event_count) { 10000 + rand(500) }
|
10
|
+
let(:parent) { "not_implemented" }
|
11
|
+
let(:config) { "not_implemented" }
|
12
|
+
let(:default_headers) {
|
13
|
+
{"Content-Type" => "application/json"}
|
14
|
+
}
|
15
|
+
subject { LogStash::Outputs::ElasticSearch.new(config) }
|
16
|
+
|
17
|
+
before do
|
18
|
+
# Add mapping and a parent document
|
19
|
+
index_url = "http://#{get_host_port()}/#{index}"
|
20
|
+
mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
|
21
|
+
Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
|
22
|
+
pdoc = { "foo" => "bar" }
|
23
|
+
Manticore.put("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers}).call
|
24
|
+
|
25
|
+
subject.register
|
26
|
+
subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
|
27
|
+
end
|
27
28
|
|
28
29
|
|
29
|
-
|
30
|
-
|
30
|
+
it "ships events" do
|
31
|
+
index_url = "http://#{get_host_port()}/#{index}"
|
31
32
|
|
32
|
-
|
33
|
+
Manticore.post("#{index_url}/_refresh").call
|
33
34
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
35
|
+
# Wait until all events are available.
|
36
|
+
Stud::try(10.times) do
|
37
|
+
query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
|
38
|
+
response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
|
39
|
+
data = response.body
|
40
|
+
result = LogStash::Json.load(data)
|
41
|
+
cur_count = result["count"]
|
42
|
+
insist { cur_count } == event_count
|
43
|
+
end
|
42
44
|
end
|
43
45
|
end
|
44
|
-
end
|
45
46
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
47
|
+
describe "(http protocol) index events with static parent" do
|
48
|
+
it_behaves_like 'a type based parent indexer' do
|
49
|
+
let(:parent) { "test" }
|
50
|
+
let(:config) {
|
51
|
+
{
|
52
|
+
"hosts" => get_host_port,
|
53
|
+
"index" => index,
|
54
|
+
"parent" => parent
|
55
|
+
}
|
54
56
|
}
|
55
|
-
|
57
|
+
end
|
56
58
|
end
|
57
|
-
end
|
58
59
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
60
|
+
describe "(http_protocol) index events with fieldref in parent value" do
|
61
|
+
it_behaves_like 'a type based parent indexer' do
|
62
|
+
let(:config) {
|
63
|
+
{
|
64
|
+
"hosts" => get_host_port,
|
65
|
+
"index" => index,
|
66
|
+
"parent" => "%{link_to}"
|
67
|
+
}
|
66
68
|
}
|
67
|
-
|
69
|
+
end
|
68
70
|
end
|
69
71
|
end
|
70
72
|
end
|
71
73
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
{
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
74
|
+
if ESHelper.es_version_satisfies?(">= 5.6")
|
75
|
+
context "when using elasticsearch 5.6 and above", :integration => true do
|
76
|
+
|
77
|
+
shared_examples "a join field based parent indexer" do
|
78
|
+
let(:index) { 10.times.collect { rand(10).to_s }.join("") }
|
79
|
+
let(:type) { 10.times.collect { rand(10).to_s }.join("") }
|
80
|
+
let(:event_count) { 10000 + rand(500) }
|
81
|
+
let(:parent) { "not_implemented" }
|
82
|
+
let(:config) { "not_implemented" }
|
83
|
+
let(:parent_id) { "test" }
|
84
|
+
let(:join_field) { "join_field" }
|
85
|
+
let(:parent_relation) { "parent_type" }
|
86
|
+
let(:child_relation) { "child_type" }
|
87
|
+
let(:default_headers) {
|
88
|
+
{"Content-Type" => "application/json"}
|
89
|
+
}
|
90
|
+
subject { LogStash::Outputs::ElasticSearch.new(config) }
|
91
|
+
|
92
|
+
before do
|
93
|
+
# Add mapping and a parent document
|
94
|
+
index_url = "http://#{get_host_port()}/#{index}"
|
95
|
+
mapping = {
|
96
|
+
"mappings" => {
|
97
|
+
type => {
|
98
|
+
"properties" => {
|
99
|
+
join_field => {
|
100
|
+
"type" => "join",
|
101
|
+
"relations" => { parent_relation => child_relation }
|
102
|
+
}
|
99
103
|
}
|
100
104
|
}
|
101
105
|
}
|
102
106
|
}
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
107
|
+
if ESHelper.es_version_satisfies?('<6')
|
108
|
+
mapping.merge!({
|
109
|
+
"settings" => {
|
110
|
+
"mapping.single_type" => true
|
111
|
+
}})
|
112
|
+
end
|
113
|
+
Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
|
114
|
+
pdoc = { "message" => "ohayo", join_field => parent_relation }
|
115
|
+
Manticore.put("#{index_url}/#{type}/#{parent_id}", {:body => pdoc.to_json, :headers => default_headers}).call
|
116
|
+
|
117
|
+
subject.register
|
118
|
+
subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => parent_id, "message" => "Hello World!", join_field => child_relation) })
|
119
|
+
end
|
111
120
|
|
112
121
|
|
113
|
-
|
114
|
-
|
122
|
+
it "ships events" do
|
123
|
+
index_url = "http://#{get_host_port()}/#{index}"
|
115
124
|
|
116
|
-
|
125
|
+
Manticore.post("#{index_url}/_refresh").call
|
117
126
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
127
|
+
# Wait until all events are available.
|
128
|
+
Stud::try(10.times) do
|
129
|
+
query = { "query" => { "has_parent" => { "parent_type" => parent_relation, "query" => { "match_all" => { } } } } }
|
130
|
+
response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
|
131
|
+
data = response.body
|
132
|
+
result = LogStash::Json.load(data)
|
133
|
+
cur_count = result["count"]
|
134
|
+
insist { cur_count } == event_count
|
135
|
+
end
|
126
136
|
end
|
127
137
|
end
|
128
|
-
end
|
129
138
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
139
|
+
describe "(http protocol) index events with static parent" do
|
140
|
+
it_behaves_like 'a join field based parent indexer' do
|
141
|
+
let(:config) {
|
142
|
+
{
|
143
|
+
"hosts" => get_host_port,
|
144
|
+
"index" => index,
|
145
|
+
"parent" => parent_id,
|
146
|
+
"document_type" => type,
|
147
|
+
"join_field" => join_field,
|
148
|
+
"manage_template" => false
|
149
|
+
}
|
140
150
|
}
|
141
|
-
|
151
|
+
end
|
142
152
|
end
|
143
|
-
end
|
144
153
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
154
|
+
describe "(http_protocol) index events with fieldref in parent value" do
|
155
|
+
it_behaves_like 'a join field based parent indexer' do
|
156
|
+
let(:config) {
|
157
|
+
{
|
158
|
+
"hosts" => get_host_port,
|
159
|
+
"index" => index,
|
160
|
+
"parent" => "%{link_to}",
|
161
|
+
"document_type" => type,
|
162
|
+
"join_field" => join_field,
|
163
|
+
"manage_template" => false
|
164
|
+
}
|
155
165
|
}
|
156
|
-
|
166
|
+
end
|
157
167
|
end
|
158
168
|
end
|
159
169
|
end
|