logstash-output-elasticsearch 11.2.0-java → 11.3.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +4 -4
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json +925 -180
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json +926 -184
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json +3695 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json +5777 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json +5782 -0
- data/lib/logstash/outputs/elasticsearch.rb +6 -1
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/integration/outputs/templates_spec.rb +100 -65
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +6 -0
- metadata +5 -2
@@ -102,7 +102,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
102
102
|
include(LogStash::Outputs::ElasticSearch::Ilm)
|
103
103
|
|
104
104
|
# ecs_compatibility option, provided by Logstash core or the support adapter.
|
105
|
-
include(LogStash::PluginMixins::ECSCompatibilitySupport)
|
105
|
+
include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
|
106
106
|
|
107
107
|
# Generic/API config options that any document indexer output needs
|
108
108
|
include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
|
@@ -300,6 +300,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
300
300
|
|
301
301
|
@bulk_request_metrics = metric.namespace(:bulk_requests)
|
302
302
|
@document_level_metrics = metric.namespace(:documents)
|
303
|
+
|
304
|
+
if ecs_compatibility == :v8
|
305
|
+
@logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
|
306
|
+
"Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
|
307
|
+
end
|
303
308
|
end
|
304
309
|
|
305
310
|
# @override post-register when ES connection established
|
@@ -1,8 +1,12 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
|
3
3
|
describe "index template expected behavior", :integration => true do
|
4
|
+
let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
|
5
|
+
|
4
6
|
subject! do
|
5
7
|
require "logstash/outputs/elasticsearch"
|
8
|
+
allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
9
|
+
|
6
10
|
settings = {
|
7
11
|
"manage_template" => true,
|
8
12
|
"template_overwrite" => true,
|
@@ -11,86 +15,117 @@ describe "index template expected behavior", :integration => true do
|
|
11
15
|
next LogStash::Outputs::ElasticSearch.new(settings)
|
12
16
|
end
|
13
17
|
|
14
|
-
|
15
|
-
# Delete all templates first.
|
16
|
-
require "elasticsearch"
|
18
|
+
let(:elasticsearch_client) { get_client }
|
17
19
|
|
18
|
-
|
19
|
-
|
20
|
-
|
20
|
+
before(:each) do
|
21
|
+
# delete indices and templates
|
22
|
+
require "elasticsearch"
|
21
23
|
|
24
|
+
elasticsearch_client.indices.delete_template(:name => '*')
|
22
25
|
# This can fail if there are no indexes, ignore failure.
|
23
|
-
|
24
|
-
|
25
|
-
subject.register
|
26
|
-
|
27
|
-
subject.multi_receive([
|
28
|
-
LogStash::Event.new("message" => "sample message here"),
|
29
|
-
LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
|
30
|
-
LogStash::Event.new("somevalue" => 100),
|
31
|
-
LogStash::Event.new("somevalue" => 10),
|
32
|
-
LogStash::Event.new("somevalue" => 1),
|
33
|
-
LogStash::Event.new("country" => "us"),
|
34
|
-
LogStash::Event.new("country" => "at"),
|
35
|
-
LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
|
36
|
-
])
|
37
|
-
|
38
|
-
@es.indices.refresh
|
39
|
-
|
40
|
-
# Wait or fail until everything's indexed.
|
41
|
-
Stud::try(20.times) do
|
42
|
-
r = @es.search(index: 'logstash-*')
|
43
|
-
expect(r).to have_hits(8)
|
44
|
-
end
|
26
|
+
elasticsearch_client.indices.delete(:index => '*') rescue nil
|
45
27
|
end
|
46
28
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
29
|
+
context 'with ecs_compatibility => disabled' do
|
30
|
+
let(:ecs_compatibility) { :disabled }
|
31
|
+
before :each do
|
32
|
+
@es = elasticsearch_client # cache as ivar for tests...
|
33
|
+
|
34
|
+
subject.register
|
35
|
+
|
36
|
+
subject.multi_receive([
|
37
|
+
LogStash::Event.new("message" => "sample message here"),
|
38
|
+
LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
|
39
|
+
LogStash::Event.new("somevalue" => 100),
|
40
|
+
LogStash::Event.new("somevalue" => 10),
|
41
|
+
LogStash::Event.new("somevalue" => 1),
|
42
|
+
LogStash::Event.new("country" => "us"),
|
43
|
+
LogStash::Event.new("country" => "at"),
|
44
|
+
LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
|
45
|
+
])
|
46
|
+
|
47
|
+
@es.indices.refresh
|
48
|
+
|
49
|
+
# Wait or fail until everything's indexed.
|
50
|
+
Stud::try(20.times) do
|
51
|
+
r = @es.search(index: 'logstash-*')
|
52
|
+
expect(r).to have_hits(8)
|
53
|
+
end
|
54
|
+
end
|
52
55
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
+
it "permits phrase searching on string fields" do
|
57
|
+
results = @es.search(:q => "message:\"sample message\"")
|
58
|
+
expect(results).to have_hits(1)
|
59
|
+
expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
|
60
|
+
end
|
56
61
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
expect(values).to_not include(1)
|
61
|
-
end
|
62
|
+
it "numbers dynamically map to a numeric type and permit range queries" do
|
63
|
+
results = @es.search(:q => "somevalue:[5 TO 105]")
|
64
|
+
expect(results).to have_hits(2)
|
62
65
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
66
|
+
values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
|
67
|
+
expect(values).to include(10)
|
68
|
+
expect(values).to include(100)
|
69
|
+
expect(values).to_not include(1)
|
70
|
+
end
|
67
71
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
+
it "does not create .keyword field for top-level message field" do
|
73
|
+
results = @es.search(:q => "message.keyword:\"sample message here\"")
|
74
|
+
expect(results).to have_hits(0)
|
75
|
+
end
|
72
76
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
+
it "creates .keyword field for nested message fields" do
|
78
|
+
results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
|
79
|
+
expect(results).to have_hits(1)
|
80
|
+
end
|
77
81
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
+
it "creates .keyword field from any string field which is not_analyzed" do
|
83
|
+
results = @es.search(:q => "country.keyword:\"us\"")
|
84
|
+
expect(results).to have_hits(1)
|
85
|
+
expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
|
86
|
+
|
87
|
+
# partial or terms should not work.
|
88
|
+
results = @es.search(:q => "country.keyword:\"u\"")
|
89
|
+
expect(results).to have_hits(0)
|
90
|
+
end
|
91
|
+
|
92
|
+
it "make [geoip][location] a geo_point" do
|
93
|
+
expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
|
94
|
+
end
|
95
|
+
|
96
|
+
it "aggregate .keyword results correctly " do
|
97
|
+
results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
|
98
|
+
terms = results["buckets"].collect { |b| b["key"] }
|
82
99
|
|
83
|
-
|
84
|
-
|
100
|
+
expect(terms).to include("us")
|
101
|
+
|
102
|
+
# 'at' is a stopword, make sure stopwords are not ignored.
|
103
|
+
expect(terms).to include("at")
|
104
|
+
end
|
85
105
|
end
|
86
106
|
|
87
|
-
|
88
|
-
|
89
|
-
|
107
|
+
context 'with ECS enabled' do
|
108
|
+
let(:ecs_compatibility) { :v1 }
|
109
|
+
|
110
|
+
before(:each) do
|
111
|
+
subject.register # should load template?
|
112
|
+
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
113
|
+
end
|
90
114
|
|
91
|
-
|
115
|
+
let(:elasticsearch_cluster_major_version) do
|
116
|
+
elasticsearch_client.info&.dig("version", "number" )&.split('.')&.map(&:to_i)&.first
|
117
|
+
end
|
92
118
|
|
93
|
-
|
94
|
-
|
119
|
+
it 'loads the templates' do
|
120
|
+
aggregate_failures do
|
121
|
+
if elasticsearch_cluster_major_version >= 8
|
122
|
+
# In ES 8+ we use the _index_template API
|
123
|
+
expect(elasticsearch_client.indices.exists_index_template(name: 'ecs-logstash')).to be_truthy
|
124
|
+
else
|
125
|
+
# Otherwise, we used the legacy _template API
|
126
|
+
expect(elasticsearch_client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
95
130
|
end
|
96
131
|
end
|
@@ -27,6 +27,12 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
+
context 'when ECS v8 is requested' do
|
31
|
+
it 'resolves' do
|
32
|
+
expect(described_class.default_template_path(7, :v8)).to end_with("/templates/ecs-v8/elasticsearch-7x.json")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
30
36
|
describe "index template with ilm settings" do
|
31
37
|
let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
|
32
38
|
let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 11.
|
4
|
+
version: 11.3.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-11-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -212,6 +212,9 @@ files:
|
|
212
212
|
- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
|
213
213
|
- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
|
214
214
|
- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
|
215
|
+
- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
|
216
|
+
- lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json
|
217
|
+
- lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json
|
215
218
|
- lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
|
216
219
|
- lib/logstash/plugin_mixins/elasticsearch/common.rb
|
217
220
|
- lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb
|