logstash-output-elasticsearch 8.1.1-java → 8.2.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,18 +1,19 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
 
3
- describe "Ingest pipeline execution behavior", :integration => true, :version_greater_than_equal_to_5x => true do
4
- subject! do
5
- require "logstash/outputs/elasticsearch"
6
- settings = {
7
- "hosts" => "#{get_host_port()}",
8
- "pipeline" => "apache-logs"
9
- }
10
- next LogStash::Outputs::ElasticSearch.new(settings)
11
- end
3
+ if ESHelper.es_version_satisfies?(">= 5")
4
+ describe "Ingest pipeline execution behavior", :integration => true do
5
+ subject! do
6
+ require "logstash/outputs/elasticsearch"
7
+ settings = {
8
+ "hosts" => "#{get_host_port()}",
9
+ "pipeline" => "apache-logs"
10
+ }
11
+ next LogStash::Outputs::ElasticSearch.new(settings)
12
+ end
12
13
 
13
- let(:ftw_client) { FTW::Agent.new }
14
- let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
15
- let(:apache_logs_pipeline) { '
14
+ let(:ftw_client) { FTW::Agent.new }
15
+ let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
16
+ let(:apache_logs_pipeline) { '
16
17
  {
17
18
  "description" : "Pipeline to parse Apache logs",
18
19
  "processors" : [
@@ -24,52 +25,53 @@ describe "Ingest pipeline execution behavior", :integration => true, :version_gr
24
25
  }
25
26
  ]
26
27
  }'
27
- }
28
+ }
28
29
 
29
- before :each do
30
- # Delete all templates first.
31
- require "elasticsearch"
30
+ before :each do
31
+ # Delete all templates first.
32
+ require "elasticsearch"
32
33
 
33
- # Clean ES of data before we start.
34
- @es = get_client
35
- @es.indices.delete_template(:name => "*")
34
+ # Clean ES of data before we start.
35
+ @es = get_client
36
+ @es.indices.delete_template(:name => "*")
36
37
 
37
- # This can fail if there are no indexes, ignore failure.
38
- @es.indices.delete(:index => "*") rescue nil
38
+ # This can fail if there are no indexes, ignore failure.
39
+ @es.indices.delete(:index => "*") rescue nil
39
40
 
40
- # delete existing ingest pipeline
41
- req = ftw_client.delete(ingest_url)
42
- ftw_client.execute(req)
41
+ # delete existing ingest pipeline
42
+ req = ftw_client.delete(ingest_url)
43
+ ftw_client.execute(req)
43
44
 
44
- # register pipeline
45
- req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
46
- req.headers["Content-Type"] = "application/json"
47
- ftw_client.execute(req)
45
+ # register pipeline
46
+ req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
47
+ req.headers["Content-Type"] = "application/json"
48
+ ftw_client.execute(req)
48
49
 
49
- #TODO: Use esclient
50
- #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
50
+ #TODO: Use esclient
51
+ #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
51
52
 
52
- subject.register
53
- subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
54
- @es.indices.refresh
53
+ subject.register
54
+ subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
55
+ @es.indices.refresh
55
56
 
56
- #Wait or fail until everything's indexed.
57
- Stud::try(20.times) do
58
- r = @es.search
59
- insist { r["hits"]["total"] } == 1
57
+ #Wait or fail until everything's indexed.
58
+ Stud::try(20.times) do
59
+ r = @es.search
60
+ insist { r["hits"]["total"] } == 1
61
+ end
60
62
  end
61
- end
62
63
 
63
- it "indexes using the proper pipeline" do
64
- results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
65
- insist { results["hits"]["total"] } == 1
66
- insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
67
- insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
68
- insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
69
- insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
70
- insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
71
- insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
72
- insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
73
- insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
64
+ it "indexes using the proper pipeline" do
65
+ results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
66
+ insist { results["hits"]["total"] } == 1
67
+ insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
68
+ insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
69
+ insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
70
+ insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
71
+ insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
72
+ insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
73
+ insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
74
+ insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
75
+ end
74
76
  end
75
77
  end
@@ -1,148 +1,179 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
 
3
- describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless', :version_greater_than_equal_to_5x => true do
4
- require "logstash/outputs/elasticsearch"
5
-
6
- def get_es_output( options={} )
7
- settings = {
8
- "manage_template" => true,
9
- "index" => "logstash-update",
10
- "template_overwrite" => true,
11
- "hosts" => get_host_port(),
12
- "action" => "update",
13
- "script_lang" => "painless"
14
- }
15
- LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
16
- end
17
-
18
- before :each do
19
- @es = get_client
20
- # Delete all templates first.
21
- # Clean ES of data before we start.
22
- @es.indices.delete_template(:name => "*")
23
- # This can fail if there are no indexes, ignore failure.
24
- @es.indices.delete(:index => "*") rescue nil
25
- @es.index(
26
- :index => 'logstash-update',
27
- :type => 'logs',
28
- :id => "123",
29
- :body => { :message => 'Test', :counter => 1 }
30
- )
31
- @es.indices.refresh
32
- end
3
+ if ESHelper.es_version_satisfies?(">= 5")
4
+ describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
5
+ require "logstash/outputs/elasticsearch"
33
6
 
34
- context "scripted updates" do
35
- it "should increment a counter with event/doc 'count' variable" do
36
- subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
37
- subject.register
38
- subject.multi_receive([LogStash::Event.new("count" => 2)])
39
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
40
- insist { r["_source"]["counter"] } == 3
7
+ def get_es_output( options={} )
8
+ settings = {
9
+ "manage_template" => true,
10
+ "index" => "logstash-update",
11
+ "template_overwrite" => true,
12
+ "hosts" => get_host_port(),
13
+ "action" => "update"
14
+ }
15
+ if ESHelper.es_version_satisfies?('<6')
16
+ settings.merge!({"script_lang" => "painless"})
17
+ end
18
+ LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
41
19
  end
42
20
 
43
- it "should increment a counter with event/doc '[data][count]' nested variable" do
44
- subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
45
- subject.register
46
- subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
47
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
48
- insist { r["_source"]["counter"] } == 4
21
+ before :each do
22
+ @es = get_client
23
+ # Delete all templates first.
24
+ # Clean ES of data before we start.
25
+ @es.indices.delete_template(:name => "*")
26
+ # This can fail if there are no indexes, ignore failure.
27
+ @es.indices.delete(:index => "*") rescue nil
28
+ @es.index(
29
+ :index => 'logstash-update',
30
+ :type => 'logs',
31
+ :id => "123",
32
+ :body => { :message => 'Test', :counter => 1 }
33
+ )
34
+ @es.indices.refresh
49
35
  end
50
36
 
51
- it "should increment a counter with event/doc 'count' variable with inline script" do
52
- subject = get_es_output({
53
- 'document_id' => "123",
54
- 'script' => 'ctx._source.counter += params.event.counter',
55
- 'script_lang' => 'painless',
56
- 'script_type' => 'inline'
57
- })
58
- subject.register
59
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
60
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
61
- insist { r["_source"]["counter"] } == 4
62
- end
37
+ context "scripted updates" do
38
+ if ESHelper.es_version_satisfies?('<6')
39
+ context 'with file based scripts' do
40
+ it "should increment a counter with event/doc 'count' variable" do
41
+ subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
42
+ subject.register
43
+ subject.multi_receive([LogStash::Event.new("count" => 2)])
44
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
45
+ insist { r["_source"]["counter"] } == 3
46
+ end
63
47
 
64
- it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
65
- subject = get_es_output({
66
- 'document_id' => "123",
67
- 'doc_as_upsert' => true,
68
- 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
69
- 'script_lang' => 'painless',
70
- 'script_type' => 'inline'
71
- })
72
- subject.register
73
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
74
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
75
- insist { r["_source"]["counter"] } == 4
76
- end
48
+ it "should increment a counter with event/doc '[data][count]' nested variable" do
49
+ subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
50
+ subject.register
51
+ subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
52
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
53
+ insist { r["_source"]["counter"] } == 4
54
+ end
55
+ end
56
+ end
77
57
 
78
- it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
79
- subject = get_es_output({
80
- 'document_id' => "456",
81
- 'doc_as_upsert' => true,
82
- 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
83
- 'script_lang' => 'painless',
84
- 'script_type' => 'inline'
85
- })
86
- subject.register
87
- subject.multi_receive([LogStash::Event.new("counter" => 3 )])
88
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
89
- insist { r["_source"]["counter"] } == 3
90
- end
58
+ it "should increment a counter with event/doc 'count' variable with inline script" do
59
+ subject = get_es_output({
60
+ 'document_id' => "123",
61
+ 'script' => 'ctx._source.counter += params.event.counter',
62
+ 'script_type' => 'inline'
63
+ })
64
+ subject.register
65
+ subject.multi_receive([LogStash::Event.new("counter" => 3 )])
66
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
67
+ insist { r["_source"]["counter"] } == 4
68
+ end
91
69
 
92
- it "should increment a counter with event/doc 'count' variable with indexed script" do
93
- @es.put_script lang: 'painless', id: 'indexed_update', body: { script: 'ctx._source.counter += params.event.count' }
94
- subject = get_es_output({
95
- 'document_id' => "123",
96
- 'script' => 'indexed_update',
97
- 'script_lang' => 'painless',
98
- 'script_type' => 'indexed'
99
- })
100
- subject.register
101
- subject.multi_receive([LogStash::Event.new("count" => 4 )])
102
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
103
- insist { r["_source"]["counter"] } == 5
104
- end
105
- end
106
-
107
- context "when update with upsert" do
108
- it "should create new documents with provided upsert" do
109
- subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
110
- subject.register
111
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
112
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
113
- insist { r["_source"]["message"] } == 'upsert message'
114
- end
70
+ it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
71
+ subject = get_es_output({
72
+ 'document_id' => "123",
73
+ 'doc_as_upsert' => true,
74
+ 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
75
+ 'script_type' => 'inline'
76
+ })
77
+ subject.register
78
+ subject.multi_receive([LogStash::Event.new("counter" => 3 )])
79
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
80
+ insist { r["_source"]["counter"] } == 4
81
+ end
115
82
 
116
- it "should create new documents with event/doc as upsert" do
117
- subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
118
- subject.register
119
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
120
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
121
- insist { r["_source"]["message"] } == 'sample message here'
122
- end
83
+ it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
84
+ subject = get_es_output({
85
+ 'document_id' => "456",
86
+ 'doc_as_upsert' => true,
87
+ 'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
88
+ 'script_type' => 'inline'
89
+ })
90
+ subject.register
91
+ subject.multi_receive([LogStash::Event.new("counter" => 3 )])
92
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
93
+ insist { r["_source"]["counter"] } == 3
94
+ end
123
95
 
124
- it "should fail on documents with event/doc as upsert at external version" do
125
- subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
126
- expect { subject.register }.to raise_error(LogStash::ConfigurationError)
127
- end
128
- end
96
+ if ESHelper.es_version_satisfies?('<6')
97
+ context 'with an indexed script' do
98
+ it "should increment a counter with event/doc 'count' variable with indexed script" do
99
+ @es.put_script lang: 'painless', id: 'indexed_update', body: { script: 'ctx._source.counter += params.event.count' }
100
+ subject = get_es_output({
101
+ 'document_id' => "123",
102
+ 'script' => 'indexed_update',
103
+ 'script_type' => 'indexed'
104
+ })
105
+ subject.register
106
+ subject.multi_receive([LogStash::Event.new("count" => 4 )])
107
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
108
+ insist { r["_source"]["counter"] } == 5
109
+ end
110
+ end
111
+ end
112
+ end
113
+
114
+ context "when update with upsert" do
115
+ it "should create new documents with provided upsert" do
116
+ subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
117
+ subject.register
118
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
119
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
120
+ insist { r["_source"]["message"] } == 'upsert message'
121
+ end
122
+
123
+ it "should create new documents with event/doc as upsert" do
124
+ subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
125
+ subject.register
126
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
127
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
128
+ insist { r["_source"]["message"] } == 'sample message here'
129
+ end
129
130
 
130
- context "updates with scripted upsert" do
131
- it "should create new documents with upsert content" do
132
- subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
133
- subject.register
134
- subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
135
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
136
- insist { r["_source"]["message"] } == 'upsert message'
131
+ it "should fail on documents with event/doc as upsert at external version" do
132
+ subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
133
+ expect { subject.register }.to raise_error(LogStash::ConfigurationError)
134
+ end
137
135
  end
138
136
 
139
- it "should create new documents with event/doc as script params" do
140
- subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
141
- subject.register
142
- subject.multi_receive([LogStash::Event.new("counter" => 1)])
143
- @es.indices.refresh
144
- r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
145
- insist { r["_source"]["counter"] } == 1
137
+ context "updates with scripted upsert" do
138
+ if ESHelper.es_version_satisfies?('<6')
139
+ context 'with file based scripts' do
140
+ it "should create new documents with upsert content" do
141
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
142
+ subject.register
143
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
144
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
145
+ insist { r["_source"]["message"] } == 'upsert message'
146
+ end
147
+
148
+ it "should create new documents with event/doc as script params" do
149
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
150
+ subject.register
151
+ subject.multi_receive([LogStash::Event.new("counter" => 1)])
152
+ @es.indices.refresh
153
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
154
+ insist { r["_source"]["counter"] } == 1
155
+ end
156
+ end
157
+ end
158
+
159
+ context 'with an inline script' do
160
+ it "should create new documents with upsert content" do
161
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
162
+ subject.register
163
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
164
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
165
+ insist { r["_source"]["message"] } == 'upsert message'
166
+ end
167
+
168
+ it "should create new documents with event/doc as script params" do
169
+ subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
170
+ subject.register
171
+ subject.multi_receive([LogStash::Event.new("counter" => 1)])
172
+ @es.indices.refresh
173
+ r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
174
+ insist { r["_source"]["counter"] } == 1
175
+ end
176
+ end
146
177
  end
147
178
  end
148
179
  end
@@ -1,159 +1,169 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
  require "logstash/outputs/elasticsearch"
3
3
 
4
- context "when using elasticsearch 5.x and before", :integration => true, :version_less_than_equal_to_5x => true do
5
- shared_examples "a type based parent indexer" do
6
- let(:index) { 10.times.collect { rand(10).to_s }.join("") }
7
- let(:type) { 10.times.collect { rand(10).to_s }.join("") }
8
- let(:event_count) { 10000 + rand(500) }
9
- let(:parent) { "not_implemented" }
10
- let(:config) { "not_implemented" }
11
- let(:default_headers) {
12
- {"Content-Type" => "application/json"}
13
- }
14
- subject { LogStash::Outputs::ElasticSearch.new(config) }
15
-
16
- before do
17
- # Add mapping and a parent document
18
- index_url = "http://#{get_host_port()}/#{index}"
19
- mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
20
- Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
21
- pdoc = { "foo" => "bar" }
22
- Manticore.put("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers}).call
23
-
24
- subject.register
25
- subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
26
- end
4
+ if ESHelper.es_version_satisfies?("<= 5.x")
5
+ context "when using elasticsearch 5.x and before", :integration => true do
6
+ shared_examples "a type based parent indexer" do
7
+ let(:index) { 10.times.collect { rand(10).to_s }.join("") }
8
+ let(:type) { 10.times.collect { rand(10).to_s }.join("") }
9
+ let(:event_count) { 10000 + rand(500) }
10
+ let(:parent) { "not_implemented" }
11
+ let(:config) { "not_implemented" }
12
+ let(:default_headers) {
13
+ {"Content-Type" => "application/json"}
14
+ }
15
+ subject { LogStash::Outputs::ElasticSearch.new(config) }
16
+
17
+ before do
18
+ # Add mapping and a parent document
19
+ index_url = "http://#{get_host_port()}/#{index}"
20
+ mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
21
+ Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
22
+ pdoc = { "foo" => "bar" }
23
+ Manticore.put("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers}).call
24
+
25
+ subject.register
26
+ subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
27
+ end
27
28
 
28
29
 
29
- it "ships events" do
30
- index_url = "http://#{get_host_port()}/#{index}"
30
+ it "ships events" do
31
+ index_url = "http://#{get_host_port()}/#{index}"
31
32
 
32
- Manticore.post("#{index_url}/_refresh").call
33
+ Manticore.post("#{index_url}/_refresh").call
33
34
 
34
- # Wait until all events are available.
35
- Stud::try(10.times) do
36
- query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
37
- response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
38
- data = response.body
39
- result = LogStash::Json.load(data)
40
- cur_count = result["count"]
41
- insist { cur_count } == event_count
35
+ # Wait until all events are available.
36
+ Stud::try(10.times) do
37
+ query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
38
+ response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
39
+ data = response.body
40
+ result = LogStash::Json.load(data)
41
+ cur_count = result["count"]
42
+ insist { cur_count } == event_count
43
+ end
42
44
  end
43
45
  end
44
- end
45
46
 
46
- describe "(http protocol) index events with static parent" do
47
- it_behaves_like 'a type based parent indexer' do
48
- let(:parent) { "test" }
49
- let(:config) {
50
- {
51
- "hosts" => get_host_port,
52
- "index" => index,
53
- "parent" => parent
47
+ describe "(http protocol) index events with static parent" do
48
+ it_behaves_like 'a type based parent indexer' do
49
+ let(:parent) { "test" }
50
+ let(:config) {
51
+ {
52
+ "hosts" => get_host_port,
53
+ "index" => index,
54
+ "parent" => parent
55
+ }
54
56
  }
55
- }
57
+ end
56
58
  end
57
- end
58
59
 
59
- describe "(http_protocol) index events with fieldref in parent value" do
60
- it_behaves_like 'a type based parent indexer' do
61
- let(:config) {
62
- {
63
- "hosts" => get_host_port,
64
- "index" => index,
65
- "parent" => "%{link_to}"
60
+ describe "(http_protocol) index events with fieldref in parent value" do
61
+ it_behaves_like 'a type based parent indexer' do
62
+ let(:config) {
63
+ {
64
+ "hosts" => get_host_port,
65
+ "index" => index,
66
+ "parent" => "%{link_to}"
67
+ }
66
68
  }
67
- }
69
+ end
68
70
  end
69
71
  end
70
72
  end
71
73
 
72
- context "when using elasticsearch 6.x and above", :integration => true, :version_greater_than_equal_to_6x => true do
73
-
74
- shared_examples "a join field based parent indexer" do
75
- let(:index) { 10.times.collect { rand(10).to_s }.join("") }
76
- let(:type) { 10.times.collect { rand(10).to_s }.join("") }
77
- let(:event_count) { 10000 + rand(500) }
78
- let(:parent) { "not_implemented" }
79
- let(:config) { "not_implemented" }
80
- let(:parent_id) { "test" }
81
- let(:join_field) { "join_field" }
82
- let(:parent_relation) { "parent_type" }
83
- let(:child_relation) { "child_type" }
84
- let(:default_headers) {
85
- {"Content-Type" => "application/json"}
86
- }
87
- subject { LogStash::Outputs::ElasticSearch.new(config) }
88
-
89
- before do
90
- # Add mapping and a parent document
91
- index_url = "http://#{get_host_port()}/#{index}"
92
- mapping = {
93
- "mappings" => {
94
- type => {
95
- "properties" => {
96
- join_field => {
97
- "type" => "join",
98
- "relations" => { parent_relation => child_relation }
74
+ if ESHelper.es_version_satisfies?(">= 5.6")
75
+ context "when using elasticsearch 5.6 and above", :integration => true do
76
+
77
+ shared_examples "a join field based parent indexer" do
78
+ let(:index) { 10.times.collect { rand(10).to_s }.join("") }
79
+ let(:type) { 10.times.collect { rand(10).to_s }.join("") }
80
+ let(:event_count) { 10000 + rand(500) }
81
+ let(:parent) { "not_implemented" }
82
+ let(:config) { "not_implemented" }
83
+ let(:parent_id) { "test" }
84
+ let(:join_field) { "join_field" }
85
+ let(:parent_relation) { "parent_type" }
86
+ let(:child_relation) { "child_type" }
87
+ let(:default_headers) {
88
+ {"Content-Type" => "application/json"}
89
+ }
90
+ subject { LogStash::Outputs::ElasticSearch.new(config) }
91
+
92
+ before do
93
+ # Add mapping and a parent document
94
+ index_url = "http://#{get_host_port()}/#{index}"
95
+ mapping = {
96
+ "mappings" => {
97
+ type => {
98
+ "properties" => {
99
+ join_field => {
100
+ "type" => "join",
101
+ "relations" => { parent_relation => child_relation }
102
+ }
99
103
  }
100
104
  }
101
105
  }
102
106
  }
103
- }
104
- Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
105
- pdoc = { "message" => "ohayo", join_field => parent_relation }
106
- Manticore.put("#{index_url}/#{type}/#{parent_id}", {:body => pdoc.to_json, :headers => default_headers}).call
107
-
108
- subject.register
109
- subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => parent_id, "message" => "Hello World!", join_field => child_relation) })
110
- end
107
+ if ESHelper.es_version_satisfies?('<6')
108
+ mapping.merge!({
109
+ "settings" => {
110
+ "mapping.single_type" => true
111
+ }})
112
+ end
113
+ Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
114
+ pdoc = { "message" => "ohayo", join_field => parent_relation }
115
+ Manticore.put("#{index_url}/#{type}/#{parent_id}", {:body => pdoc.to_json, :headers => default_headers}).call
116
+
117
+ subject.register
118
+ subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => parent_id, "message" => "Hello World!", join_field => child_relation) })
119
+ end
111
120
 
112
121
 
113
- it "ships events" do
114
- index_url = "http://#{get_host_port()}/#{index}"
122
+ it "ships events" do
123
+ index_url = "http://#{get_host_port()}/#{index}"
115
124
 
116
- Manticore.post("#{index_url}/_refresh").call
125
+ Manticore.post("#{index_url}/_refresh").call
117
126
 
118
- # Wait until all events are available.
119
- Stud::try(10.times) do
120
- query = { "query" => { "has_parent" => { "parent_type" => parent_relation, "query" => { "match_all" => { } } } } }
121
- response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
122
- data = response.body
123
- result = LogStash::Json.load(data)
124
- cur_count = result["count"]
125
- insist { cur_count } == event_count
127
+ # Wait until all events are available.
128
+ Stud::try(10.times) do
129
+ query = { "query" => { "has_parent" => { "parent_type" => parent_relation, "query" => { "match_all" => { } } } } }
130
+ response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
131
+ data = response.body
132
+ result = LogStash::Json.load(data)
133
+ cur_count = result["count"]
134
+ insist { cur_count } == event_count
135
+ end
126
136
  end
127
137
  end
128
- end
129
138
 
130
- describe "(http protocol) index events with static parent" do
131
- it_behaves_like 'a join field based parent indexer' do
132
- let(:config) {
133
- {
134
- "hosts" => get_host_port,
135
- "index" => index,
136
- "parent" => parent_id,
137
- "document_type" => type,
138
- "join_field" => join_field,
139
- "manage_template" => false
139
+ describe "(http protocol) index events with static parent" do
140
+ it_behaves_like 'a join field based parent indexer' do
141
+ let(:config) {
142
+ {
143
+ "hosts" => get_host_port,
144
+ "index" => index,
145
+ "parent" => parent_id,
146
+ "document_type" => type,
147
+ "join_field" => join_field,
148
+ "manage_template" => false
149
+ }
140
150
  }
141
- }
151
+ end
142
152
  end
143
- end
144
153
 
145
- describe "(http_protocol) index events with fieldref in parent value" do
146
- it_behaves_like 'a join field based parent indexer' do
147
- let(:config) {
148
- {
149
- "hosts" => get_host_port,
150
- "index" => index,
151
- "parent" => "%{link_to}",
152
- "document_type" => type,
153
- "join_field" => join_field,
154
- "manage_template" => false
154
+ describe "(http_protocol) index events with fieldref in parent value" do
155
+ it_behaves_like 'a join field based parent indexer' do
156
+ let(:config) {
157
+ {
158
+ "hosts" => get_host_port,
159
+ "index" => index,
160
+ "parent" => "%{link_to}",
161
+ "document_type" => type,
162
+ "join_field" => join_field,
163
+ "manage_template" => false
164
+ }
155
165
  }
156
- }
166
+ end
157
167
  end
158
168
  end
159
169
  end