logstash-output-elasticsearch 11.2.0-java → 11.3.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -102,7 +102,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
102
102
  include(LogStash::Outputs::ElasticSearch::Ilm)
103
103
 
104
104
  # ecs_compatibility option, provided by Logstash core or the support adapter.
105
- include(LogStash::PluginMixins::ECSCompatibilitySupport)
105
+ include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
106
106
 
107
107
  # Generic/API config options that any document indexer output needs
108
108
  include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
@@ -300,6 +300,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
300
300
 
301
301
  @bulk_request_metrics = metric.namespace(:bulk_requests)
302
302
  @document_level_metrics = metric.namespace(:documents)
303
+
304
+ if ecs_compatibility == :v8
305
+ @logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
306
+ "Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
307
+ end
303
308
  end
304
309
 
305
310
  # @override post-register when ES connection established
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '11.2.0'
3
+ s.version = '11.3.0'
4
4
 
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Stores logs in Elasticsearch"
@@ -1,8 +1,12 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
2
 
3
3
  describe "index template expected behavior", :integration => true do
4
+ let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
5
+
4
6
  subject! do
5
7
  require "logstash/outputs/elasticsearch"
8
+ allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
9
+
6
10
  settings = {
7
11
  "manage_template" => true,
8
12
  "template_overwrite" => true,
@@ -11,86 +15,117 @@ describe "index template expected behavior", :integration => true do
11
15
  next LogStash::Outputs::ElasticSearch.new(settings)
12
16
  end
13
17
 
14
- before :each do
15
- # Delete all templates first.
16
- require "elasticsearch"
18
+ let(:elasticsearch_client) { get_client }
17
19
 
18
- # Clean ES of data before we start.
19
- @es = get_client
20
- @es.indices.delete_template(:name => "*")
20
+ before(:each) do
21
+ # delete indices and templates
22
+ require "elasticsearch"
21
23
 
24
+ elasticsearch_client.indices.delete_template(:name => '*')
22
25
  # This can fail if there are no indexes, ignore failure.
23
- @es.indices.delete(:index => "*") rescue nil
24
-
25
- subject.register
26
-
27
- subject.multi_receive([
28
- LogStash::Event.new("message" => "sample message here"),
29
- LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
30
- LogStash::Event.new("somevalue" => 100),
31
- LogStash::Event.new("somevalue" => 10),
32
- LogStash::Event.new("somevalue" => 1),
33
- LogStash::Event.new("country" => "us"),
34
- LogStash::Event.new("country" => "at"),
35
- LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
36
- ])
37
-
38
- @es.indices.refresh
39
-
40
- # Wait or fail until everything's indexed.
41
- Stud::try(20.times) do
42
- r = @es.search(index: 'logstash-*')
43
- expect(r).to have_hits(8)
44
- end
26
+ elasticsearch_client.indices.delete(:index => '*') rescue nil
45
27
  end
46
28
 
47
- it "permits phrase searching on string fields" do
48
- results = @es.search(:q => "message:\"sample message\"")
49
- expect(results).to have_hits(1)
50
- expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
51
- end
29
+ context 'with ecs_compatibility => disabled' do
30
+ let(:ecs_compatibility) { :disabled }
31
+ before :each do
32
+ @es = elasticsearch_client # cache as ivar for tests...
33
+
34
+ subject.register
35
+
36
+ subject.multi_receive([
37
+ LogStash::Event.new("message" => "sample message here"),
38
+ LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
39
+ LogStash::Event.new("somevalue" => 100),
40
+ LogStash::Event.new("somevalue" => 10),
41
+ LogStash::Event.new("somevalue" => 1),
42
+ LogStash::Event.new("country" => "us"),
43
+ LogStash::Event.new("country" => "at"),
44
+ LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
45
+ ])
46
+
47
+ @es.indices.refresh
48
+
49
+ # Wait or fail until everything's indexed.
50
+ Stud::try(20.times) do
51
+ r = @es.search(index: 'logstash-*')
52
+ expect(r).to have_hits(8)
53
+ end
54
+ end
52
55
 
53
- it "numbers dynamically map to a numeric type and permit range queries" do
54
- results = @es.search(:q => "somevalue:[5 TO 105]")
55
- expect(results).to have_hits(2)
56
+ it "permits phrase searching on string fields" do
57
+ results = @es.search(:q => "message:\"sample message\"")
58
+ expect(results).to have_hits(1)
59
+ expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
60
+ end
56
61
 
57
- values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
58
- expect(values).to include(10)
59
- expect(values).to include(100)
60
- expect(values).to_not include(1)
61
- end
62
+ it "numbers dynamically map to a numeric type and permit range queries" do
63
+ results = @es.search(:q => "somevalue:[5 TO 105]")
64
+ expect(results).to have_hits(2)
62
65
 
63
- it "does not create .keyword field for top-level message field" do
64
- results = @es.search(:q => "message.keyword:\"sample message here\"")
65
- expect(results).to have_hits(0)
66
- end
66
+ values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
67
+ expect(values).to include(10)
68
+ expect(values).to include(100)
69
+ expect(values).to_not include(1)
70
+ end
67
71
 
68
- it "creates .keyword field for nested message fields" do
69
- results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
70
- expect(results).to have_hits(1)
71
- end
72
+ it "does not create .keyword field for top-level message field" do
73
+ results = @es.search(:q => "message.keyword:\"sample message here\"")
74
+ expect(results).to have_hits(0)
75
+ end
72
76
 
73
- it "creates .keyword field from any string field which is not_analyzed" do
74
- results = @es.search(:q => "country.keyword:\"us\"")
75
- expect(results).to have_hits(1)
76
- expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
77
+ it "creates .keyword field for nested message fields" do
78
+ results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
79
+ expect(results).to have_hits(1)
80
+ end
77
81
 
78
- # partial or terms should not work.
79
- results = @es.search(:q => "country.keyword:\"u\"")
80
- expect(results).to have_hits(0)
81
- end
82
+ it "creates .keyword field from any string field which is not_analyzed" do
83
+ results = @es.search(:q => "country.keyword:\"us\"")
84
+ expect(results).to have_hits(1)
85
+ expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
86
+
87
+ # partial or terms should not work.
88
+ results = @es.search(:q => "country.keyword:\"u\"")
89
+ expect(results).to have_hits(0)
90
+ end
91
+
92
+ it "make [geoip][location] a geo_point" do
93
+ expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
94
+ end
95
+
96
+ it "aggregate .keyword results correctly " do
97
+ results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
98
+ terms = results["buckets"].collect { |b| b["key"] }
82
99
 
83
- it "make [geoip][location] a geo_point" do
84
- expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
100
+ expect(terms).to include("us")
101
+
102
+ # 'at' is a stopword, make sure stopwords are not ignored.
103
+ expect(terms).to include("at")
104
+ end
85
105
  end
86
106
 
87
- it "aggregate .keyword results correctly " do
88
- results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
89
- terms = results["buckets"].collect { |b| b["key"] }
107
+ context 'with ECS enabled' do
108
+ let(:ecs_compatibility) { :v1 }
109
+
110
+ before(:each) do
111
+ subject.register # should load template?
112
+ subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
113
+ end
90
114
 
91
- expect(terms).to include("us")
115
+ let(:elasticsearch_cluster_major_version) do
116
+ elasticsearch_client.info&.dig("version", "number" )&.split('.')&.map(&:to_i)&.first
117
+ end
92
118
 
93
- # 'at' is a stopword, make sure stopwords are not ignored.
94
- expect(terms).to include("at")
119
+ it 'loads the templates' do
120
+ aggregate_failures do
121
+ if elasticsearch_cluster_major_version >= 8
122
+ # In ES 8+ we use the _index_template API
123
+ expect(elasticsearch_client.indices.exists_index_template(name: 'ecs-logstash')).to be_truthy
124
+ else
125
+ # Otherwise, we used the legacy _template API
126
+ expect(elasticsearch_client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
127
+ end
128
+ end
129
+ end
95
130
  end
96
131
  end
@@ -27,6 +27,12 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
27
27
  end
28
28
  end
29
29
 
30
+ context 'when ECS v8 is requested' do
31
+ it 'resolves' do
32
+ expect(described_class.default_template_path(7, :v8)).to end_with("/templates/ecs-v8/elasticsearch-7x.json")
33
+ end
34
+ end
35
+
30
36
  describe "index template with ilm settings" do
31
37
  let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
32
38
  let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.2.0
4
+ version: 11.3.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-10-12 00:00:00.000000000 Z
11
+ date: 2021-11-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -212,6 +212,9 @@ files:
212
212
  - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
213
213
  - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
214
214
  - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
215
+ - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
216
+ - lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json
217
+ - lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json
215
218
  - lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
216
219
  - lib/logstash/plugin_mixins/elasticsearch/common.rb
217
220
  - lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb