logstash-output-elasticsearch 11.11.0-java → 11.12.1-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6c35d86223e3353d75bc09c1e86ac45b164133ed877b64496e5734627f8bfe9b
4
- data.tar.gz: 57bbcc13083b42c010c13fd51466ac9872ab09522ac4fb8e2833f429f1d18fbf
3
+ metadata.gz: 484fe25cf95173ddd0bc6107393004e4b951b7302e01ef3941b8db7b2f948604
4
+ data.tar.gz: 0a63851b6e5623c2eaddbc4f3b225ec6f5a21f5c49cd3004adb753c77bb08099
5
5
  SHA512:
6
- metadata.gz: adfd2a4f7d288019fff8908bfd06a1e34ad5d07ca0bd1ca31d73d5394eddf2d8b0dfdbd0f8da983f5e66022d8d81c7852070be78914442cdc1b36fa7b67dc4ac
7
- data.tar.gz: a99011dbe58769d6b9069c8ac5a4afc5b5a3682a6226d49b1d7b3d1ca687cbbabbfcd6816c68a27027b2b315a8501efdbba43013e980191b8cb5ac3ba78a56cb
6
+ metadata.gz: db0556955eab387ce9f65784266dd11be8d1c8d2d23c10cb7fac51261068c20e12b335b4db92c8dd4741948fb34dd80be618a63245e3698357632e51e69ea3c5
7
+ data.tar.gz: 82dd5eac05b580e74e745c59204e100a463c03d16b40862e532acc0769e8f917cc576808dfbf85f2a2a9b9a892de0ab59b7a5a339ffc612ebd14051732d74b5b
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 11.12.1
2
+ - Log bulk request response body on error, not just when debug logging is enabled [#1096](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1096)
3
+
4
+ ## 11.12.0
5
+ - Add legacy template API support for Elasticsearch 8 [#1092](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1092)
6
+
1
7
  ## 11.11.0
2
8
  - When using an `api_key` along with either `cloud_id` or https `hosts`, you no longer need to also specify `ssl => true` [#1065](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1065)
3
9
 
data/docs/index.asciidoc CHANGED
@@ -362,6 +362,7 @@ This plugin supports the following configuration options plus the
362
362
  | <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
363
363
  | <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
364
364
  | <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
365
+ | <<plugins-{type}s-{plugin}-template_api>> |<<string,string>>, one of `["auto", "legacy", "composable"]`|No
365
366
  | <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
366
367
  | <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
367
368
  | <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
@@ -1080,6 +1081,22 @@ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.
1080
1081
  You can set the path to your own template here, if you so desire.
1081
1082
  If not set, the included template will be used.
1082
1083
 
1084
+ [id="plugins-{type}s-{plugin}-template_api"]
1085
+ ===== `template_api`
1086
+
1087
+ * Value can be any of: `auto`, `legacy`, `composable`
1088
+ * Default value is `auto`
1089
+
1090
+ The default setting of `auto` will use
1091
+ {ref}/index-templates.html[index template API] to create index template, if the
1092
+ Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher,
1093
+ and use {ref}/indices-templates-v1.html[legacy template API] otherwise.
1094
+
1095
+ Setting this flag to `legacy` will use legacy template API to create index template.
1096
+ Setting this flag to `composable` will use index template API to create index template.
1097
+
1098
+ NOTE: The format of template provided to <<plugins-{type}s-{plugin}-template>> needs to match the template API being used.
1099
+
1083
1100
  [id="plugins-{type}s-{plugin}-template_name"]
1084
1101
  ===== `template_name`
1085
1102
 
@@ -77,12 +77,12 @@ module LogStash; module Outputs; class ElasticSearch;
77
77
  }
78
78
  end
79
79
 
80
- def template_install(name, template, force=false)
81
- if template_exists?(name) && !force
80
+ def template_install(template_endpoint, name, template, force=false)
81
+ if template_exists?(template_endpoint, name) && !force
82
82
  @logger.debug("Found existing Elasticsearch template, skipping template management", name: name)
83
83
  return
84
84
  end
85
- template_put(name, template)
85
+ template_put(template_endpoint, name, template)
86
86
  end
87
87
 
88
88
  def last_es_version
@@ -402,20 +402,16 @@ module LogStash; module Outputs; class ElasticSearch;
402
402
  response.code >= 200 && response.code <= 299
403
403
  end
404
404
 
405
- def template_exists?(name)
405
+ def template_exists?(template_endpoint, name)
406
406
  exists?("/#{template_endpoint}/#{name}")
407
407
  end
408
408
 
409
- def template_put(name, template)
409
+ def template_put(template_endpoint, name, template)
410
410
  path = "#{template_endpoint}/#{name}"
411
411
  logger.info("Installing Elasticsearch template", name: name)
412
412
  @pool.put(path, nil, LogStash::Json.dump(template))
413
413
  end
414
414
 
415
- def template_endpoint
416
- maximum_seen_major_version < 8 ? '_template' : '_index_template'
417
- end
418
-
419
415
  # ILM methods
420
416
 
421
417
  # check whether rollover alias already exists
@@ -1,8 +1,20 @@
1
1
  module LogStash; module Outputs; class ElasticSearch
2
2
  class TemplateManager
3
+ LEGACY_TEMPLATE_ENDPOINT = '_template'.freeze
4
+ INDEX_TEMPLATE_ENDPOINT = '_index_template'.freeze
5
+
3
6
  # To be mixed into the elasticsearch plugin base
4
7
  def self.install_template(plugin)
5
8
  return unless plugin.manage_template
9
+
10
+ if plugin.maximum_seen_major_version < 8 && plugin.template_api == 'auto'
11
+ plugin.logger.warn("`template_api => auto` resolved to `legacy` since we are connected to " + "Elasticsearch #{plugin.maximum_seen_major_version}, " +
12
+ "but will resolve to `composable` the first time it connects to Elasticsearch 8+. " +
13
+ "We recommend either setting `template_api => legacy` to continue providing legacy-style templates, " +
14
+ "or migrating your template to the composable style and setting `template_api => composable`. " +
15
+ "The legacy template API is slated for removal in Elasticsearch 9.")
16
+ end
17
+
6
18
  if plugin.template
7
19
  plugin.logger.info("Using mapping template from", :path => plugin.template)
8
20
  template = read_template_file(plugin.template)
@@ -14,7 +26,7 @@ module LogStash; module Outputs; class ElasticSearch
14
26
 
15
27
  add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
16
28
  plugin.logger.debug("Attempting to install template", template: template)
17
- install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
29
+ install(plugin.client, template_endpoint(plugin), template_name(plugin), template, plugin.template_overwrite)
18
30
  end
19
31
 
20
32
  private
@@ -25,8 +37,8 @@ module LogStash; module Outputs; class ElasticSearch
25
37
  fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
26
38
  end
27
39
 
28
- def self.install(client, template_name, template, template_overwrite)
29
- client.template_install(template_name, template, template_overwrite)
40
+ def self.install(client, template_endpoint, template_name, template, template_overwrite)
41
+ client.template_install(template_endpoint, template_name, template, template_overwrite)
30
42
  end
31
43
 
32
44
  def self.add_ilm_settings_to_template(plugin, template)
@@ -63,5 +75,16 @@ module LogStash; module Outputs; class ElasticSearch
63
75
  template_data = ::IO.read(template_path)
64
76
  LogStash::Json.load(template_data)
65
77
  end
78
+
79
+ def self.template_endpoint(plugin)
80
+ if plugin.template_api == 'auto'
81
+ plugin.maximum_seen_major_version < 8 ? LEGACY_TEMPLATE_ENDPOINT : INDEX_TEMPLATE_ENDPOINT
82
+ elsif plugin.template_api.to_s == 'legacy'
83
+ LEGACY_TEMPLATE_ENDPOINT
84
+ else
85
+ INDEX_TEMPLATE_ENDPOINT
86
+ end
87
+ end
88
+
66
89
  end
67
90
  end end end
@@ -185,6 +185,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
185
185
  # the "logstash" template (i.e. removing all customized settings)
186
186
  config :template_overwrite, :validate => :boolean, :default => false
187
187
 
188
+ # Flag for enabling legacy template api for Elasticsearch 8
189
+ # Default auto will use index template api for Elasticsearch 8 and use legacy api for 7
190
+ # Set to legacy to use legacy template api
191
+ config :template_api, :validate => ['auto', 'legacy', 'composable'], :default => 'auto'
192
+
188
193
  # The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
189
194
  # See https://www.elastic.co/blog/elasticsearch-versioning-support.
190
195
  config :version, :validate => :string
@@ -335,8 +335,8 @@ module LogStash; module PluginMixins; module ElasticSearch
335
335
  retry unless @stopping.true?
336
336
  rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
337
337
  @bulk_request_metrics.increment(:failures)
338
- log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s, :content_length => e.request_body.bytesize}
339
- log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
338
+ log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s,
339
+ :content_length => e.request_body.bytesize, :body => e.response_body}
340
340
  message = "Encountered a retryable error (will retry with exponential backoff)"
341
341
 
342
342
  # We treat 429s as a special case because these really aren't errors, but
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '11.11.0'
3
+ s.version = '11.12.1'
4
4
  s.licenses = ['apache-2.0']
5
5
  s.summary = "Stores logs in Elasticsearch"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -140,27 +140,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
140
140
  end
141
141
  end
142
142
 
143
- describe "index template" do
144
- subject { described_class.new(base_options) }
145
- let(:template_name) { "logstash" }
146
- let(:template) { {} }
147
- let(:get_response) {
148
- double("response", :body => {})
149
- }
150
-
151
- it "should call composable index template in version 8+" do
152
- expect(subject).to receive(:maximum_seen_major_version).and_return(8)
153
- expect(subject.pool).to receive(:put).with("_index_template/#{template_name}", nil, anything).and_return(get_response)
154
- subject.template_put(template_name, template)
155
- end
156
-
157
- it "should call index template in version < 8" do
158
- expect(subject).to receive(:maximum_seen_major_version).and_return(7)
159
- expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
160
- subject.template_put(template_name, template)
161
- end
162
- end
163
-
164
143
  describe "join_bulk_responses" do
165
144
  subject { described_class.new(base_options) }
166
145
 
@@ -63,4 +63,53 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
63
63
  end
64
64
  end
65
65
  end
66
+
67
+ describe "template endpoint" do
68
+ describe "template_api => 'auto'" do
69
+ let(:plugin_settings) { {"manage_template" => true, "template_api" => 'auto'} }
70
+ let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
71
+
72
+ describe "in version 8+" do
73
+ it "should use index template API" do
74
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
75
+ endpoint = described_class.template_endpoint(plugin)
76
+ expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
77
+ end
78
+ end
79
+
80
+ describe "in version < 8" do
81
+ it "should use legacy template API" do
82
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
83
+ endpoint = described_class.template_endpoint(plugin)
84
+ expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
85
+ end
86
+ end
87
+ end
88
+
89
+ describe "template_api => 'legacy'" do
90
+ let(:plugin_settings) { {"manage_template" => true, "template_api" => 'legacy'} }
91
+ let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
92
+
93
+ describe "in version 8+" do
94
+ it "should use legacy template API" do
95
+ expect(plugin).to receive(:maximum_seen_major_version).never
96
+ endpoint = described_class.template_endpoint(plugin)
97
+ expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
98
+ end
99
+ end
100
+ end
101
+
102
+ describe "template_api => 'composable'" do
103
+ let(:plugin_settings) { {"manage_template" => true, "template_api" => 'composable'} }
104
+ let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
105
+
106
+ describe "in version 8+" do
107
+ it "should use legacy template API" do
108
+ expect(plugin).to receive(:maximum_seen_major_version).never
109
+ endpoint = described_class.template_endpoint(plugin)
110
+ expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager:: INDEX_TEMPLATE_ENDPOINT)
111
+ end
112
+ end
113
+ end
114
+ end
66
115
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.11.0
4
+ version: 11.12.1
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-10-12 00:00:00.000000000 Z
11
+ date: 2022-11-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement